From 03f260ebbfc61beecd41328fcecf2f355a3d82de Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Mon, 12 May 2025 16:31:15 -0700 Subject: [PATCH 1/4] Added support for insert append for direct path inserts fixing ER #174 --- Makefile | 2 +- .../incremental/strategies.sql | 19 +++++++++++++++---- .../models/us_product_delete_insert.sql | 1 + setup.cfg | 2 +- setup.py | 2 +- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index f3f845b..3b7d211 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Configuration variables -VERSION=1.9.1 +VERSION=1.9.2 PROJ_DIR?=$(shell pwd) VENV_DIR?=${PROJ_DIR}/.bldenv BUILD_DIR=${PROJ_DIR}/build diff --git a/dbt/include/oracle/macros/materializations/incremental/strategies.sql b/dbt/include/oracle/macros/materializations/incremental/strategies.sql index dad5005..8f46dca 100644 --- a/dbt/include/oracle/macros/materializations/incremental/strategies.sql +++ b/dbt/include/oracle/macros/materializations/incremental/strategies.sql @@ -94,12 +94,15 @@ {% macro oracle__get_incremental_append_sql(args_dict) %} {%- set parallel = config.get('parallel', none) -%} + {%- set insert_mode = config.get('insert_mode', none) -%} {%- set dest_columns = args_dict["dest_columns"] -%} {%- set temp_relation = args_dict["temp_relation"] -%} {%- set target_relation = args_dict["target_relation"] -%} {%- set dest_column_names = dest_columns | map(attribute='name') | list -%} {%- set dest_cols_csv = get_quoted_column_csv(model, dest_column_names) -%} - INSERT {% if parallel %} /*+PARALLEL({{ parallel }})*/ {% endif %} INTO {{ target_relation }} ({{ dest_cols_csv }}) + INSERT {% if parallel %} /*+PARALLEL({{ parallel }})*/ {% endif %} + {% if insert_mode == 'append' %} /*+ append */ {% endif %} + INTO {{ target_relation }} ({{ dest_cols_csv }}) ( SELECT {{ dest_cols_csv }} FROM {{ temp_relation }} @@ -108,6 +111,7 @@ {% macro oracle__get_incremental_merge_sql(args_dict) %} {%- set parallel = config.get('parallel', none) -%} + {%- set insert_mode = config.get('insert_mode', none) -%} {%- set dest_columns = args_dict["dest_columns"] -%} {%- set temp_relation = args_dict["temp_relation"] -%} {%- set target_relation = args_dict["target_relation"] -%} @@ -138,7 +142,9 @@ {% endfor -%} ) {%- else -%} - insert {% if parallel %} /*+parallel({{ parallel }})*/ {% endif %} into {{ target_relation }} ({{ dest_cols_csv }}) + insert {% if parallel %} /*+parallel({{ parallel }})*/ {% endif %} + {% if insert_mode == 'append' %} /*+ append */ {% endif %} + into {{ target_relation }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} from {{ temp_relation }} @@ -178,6 +184,7 @@ {% macro oracle__get_incremental_delete_insert_sql(args_dict) %} {%- set parallel = config.get('parallel', none) -%} + {%- set insert_mode = config.get('insert_mode', none) -%} {%- set dest_columns = args_dict["dest_columns"] -%} {%- set temp_relation = args_dict["temp_relation"] -%} {%- set target_relation = args_dict["target_relation"] -%} @@ -188,12 +195,16 @@ {%- if unique_key or incremental_predicates -%} BEGIN EXECUTE IMMEDIATE '{{ oracle__get_delete_sql_for_delete_insert_strategy(target_relation, temp_relation, unique_key, incremental_predicates) }}'; - EXECUTE IMMEDIATE 'insert {% if parallel %} /*+parallel({{ parallel }})*/ {% endif %} into {{ target_relation }} ({{ dest_cols_csv }})( + EXECUTE IMMEDIATE 'insert {% if parallel %} /*+parallel({{ parallel }})*/ {% endif %} + {% if insert_mode == 'append' %} /*+ append */ {% endif %} + into {{ target_relation }} ({{ dest_cols_csv }})( select {{ dest_cols_csv }} from {{ temp_relation }})'; END; {%- else -%} - insert {% if parallel %} /*+parallel({{ parallel }})*/ {% endif %} into {{ target_relation }} ({{ dest_cols_csv }}) + insert {% if parallel %} /*+parallel({{ parallel }})*/ {% endif %} + {% if insert_mode == 'append' %} /*+ append */ {% endif %} + into {{ target_relation }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} from {{ temp_relation }} diff --git a/dbt_adbs_test_project/models/us_product_delete_insert.sql b/dbt_adbs_test_project/models/us_product_delete_insert.sql index b4b516e..09513e2 100644 --- a/dbt_adbs_test_project/models/us_product_delete_insert.sql +++ b/dbt_adbs_test_project/models/us_product_delete_insert.sql @@ -20,6 +20,7 @@ incremental_predicates=["DBT_INTERNAL_DEST.calendar_month_desc > TO_CHAR(sysdate, ''yyyy/mm/dd'')"], incremental_strategy='delete+insert', parallel=4, + insert_mode="append", partition_config={"clause": "PARTITION BY HASH(PROD_NAME) PARTITIONS 4"}, table_compression_clause='COLUMN STORE COMPRESS FOR QUERY LOW') }} diff --git a/setup.cfg b/setup.cfg index 9463bb0..fe9101b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = dbt-oracle -version = 1.9.1 +version = 1.9.2 description = dbt (data build tool) adapter for Oracle Autonomous Database long_description = file: README.md long_description_content_type = text/markdown diff --git a/setup.py b/setup.py index 5df3ceb..6b3e75e 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ url = 'https://github.com/oracle/dbt-oracle' -VERSION = '1.9.1' +VERSION = '1.9.2' setup( author="Oracle", python_requires='>=3.9', From 1c55b81236b72b495c13ac43b169ec0a4c22a4bd Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Mon, 12 May 2025 17:25:20 -0700 Subject: [PATCH 2/4] fix snapshot error 'Snapshot target is missing configured columns' --- .../oracle/macros/materializations/snapshot/snapshot.sql | 6 +++--- dbt_adbs_test_project/snapshots/promotion_costs.sql | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql index 0745469..befcde7 100644 --- a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql @@ -43,7 +43,7 @@ {% macro oracle__snapshot_staging_table(strategy, source_sql, target_relation) -%} - {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} + {% set columns = config.get('snapshot_meta_column_names') or get_snapshot_table_column_names() %} {% if strategy.hard_deletes == 'new_record' %} {% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, snapshot_get_time()]) %} {% endif %} @@ -212,7 +212,7 @@ {% macro oracle__build_snapshot_table(strategy, sql) %} - {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} + {% set columns = config.get('snapshot_meta_column_names') or get_snapshot_table_column_names() %} select sbq.*, {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}, @@ -301,7 +301,7 @@ {% else %} - {% set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %} + {% set columns = config.get("snapshot_meta_column_names") or get_snapshot_table_column_names() %} {{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, columns, strategy) }} diff --git a/dbt_adbs_test_project/snapshots/promotion_costs.sql b/dbt_adbs_test_project/snapshots/promotion_costs.sql index cbaf7a2..0ab66cd 100644 --- a/dbt_adbs_test_project/snapshots/promotion_costs.sql +++ b/dbt_adbs_test_project/snapshots/promotion_costs.sql @@ -18,7 +18,12 @@ strategy='check', unique_key='promo_id', check_cols='all', - hard_deletes='invalidate' + hard_deletes='invalidate', + snapshot_meta_column_names={ + "dbt_valid_from": "promo_valid_from", + "dbt_valid_to": "promo_valid_to", + "dbt_scd_id": "dbt_scd_id" + } ) }} select * from {{ ref('promotion_costs') }} From 6cb132fc2309cf1a465a01cf397e96f27e5f9eee Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Mon, 12 May 2025 19:07:30 -0700 Subject: [PATCH 3/4] Fix model config for snapshots --- .../oracle/macros/materializations/snapshot/snapshot.sql | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql index befcde7..05c547b 100644 --- a/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/oracle/macros/materializations/snapshot/snapshot.sql @@ -43,7 +43,7 @@ {% macro oracle__snapshot_staging_table(strategy, source_sql, target_relation) -%} - {% set columns = config.get('snapshot_meta_column_names') or get_snapshot_table_column_names() %} + {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} {% if strategy.hard_deletes == 'new_record' %} {% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, snapshot_get_time()]) %} {% endif %} @@ -212,7 +212,7 @@ {% macro oracle__build_snapshot_table(strategy, sql) %} - {% set columns = config.get('snapshot_meta_column_names') or get_snapshot_table_column_names() %} + {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %} select sbq.*, {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}, @@ -259,7 +259,6 @@ {% materialization snapshot, adapter='oracle' %} - {%- set config = model['config'] -%} {%- set target_table = model.get('alias', model.get('name')) -%} @@ -301,7 +300,7 @@ {% else %} - {% set columns = config.get("snapshot_meta_column_names") or get_snapshot_table_column_names() %} + {% set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() %} {{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, columns, strategy) }} From 049f5de53d94ab800b20903faeee292d27d4f76d Mon Sep 17 00:00:00 2001 From: Abhishek Singh Date: Thu, 15 May 2025 18:27:39 -0700 Subject: [PATCH 4/4] Fixed insert hint and upgraded oracledb driver to 3.1.1 --- dbt/include/oracle/macros/adapters.sql | 10 ++++++++++ .../materializations/incremental/strategies.sql | 15 +++++++-------- requirements.txt | 2 +- setup.cfg | 2 +- setup.py | 2 +- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/dbt/include/oracle/macros/adapters.sql b/dbt/include/oracle/macros/adapters.sql index 14bcc58..34bc031 100644 --- a/dbt/include/oracle/macros/adapters.sql +++ b/dbt/include/oracle/macros/adapters.sql @@ -422,3 +422,13 @@ {% set db_name = results.columns[0].values()[0] %} {{ return(db_name) }} {% endmacro %} + +{% macro generate_insert_hint(parallel, insert_mode) %} + {% if parallel and insert_mode == 'append' %} + /*+parallel({{ parallel }}) append*/ + {% elif parallel %} + /*+parallel({{ parallel }})*/ + {% elif insert_mode == 'append' %} + /*+ append */ + {% endif %} +{% endmacro %} diff --git a/dbt/include/oracle/macros/materializations/incremental/strategies.sql b/dbt/include/oracle/macros/materializations/incremental/strategies.sql index 8f46dca..1e76829 100644 --- a/dbt/include/oracle/macros/materializations/incremental/strategies.sql +++ b/dbt/include/oracle/macros/materializations/incremental/strategies.sql @@ -95,13 +95,13 @@ {% macro oracle__get_incremental_append_sql(args_dict) %} {%- set parallel = config.get('parallel', none) -%} {%- set insert_mode = config.get('insert_mode', none) -%} + {%- set insert_hint = generate_insert_hint(parallel, insert_mode) -%} {%- set dest_columns = args_dict["dest_columns"] -%} {%- set temp_relation = args_dict["temp_relation"] -%} {%- set target_relation = args_dict["target_relation"] -%} {%- set dest_column_names = dest_columns | map(attribute='name') | list -%} {%- set dest_cols_csv = get_quoted_column_csv(model, dest_column_names) -%} - INSERT {% if parallel %} /*+PARALLEL({{ parallel }})*/ {% endif %} - {% if insert_mode == 'append' %} /*+ append */ {% endif %} + INSERT {{ insert_hint }} INTO {{ target_relation }} ({{ dest_cols_csv }}) ( SELECT {{ dest_cols_csv }} @@ -112,6 +112,7 @@ {% macro oracle__get_incremental_merge_sql(args_dict) %} {%- set parallel = config.get('parallel', none) -%} {%- set insert_mode = config.get('insert_mode', none) -%} + {%- set insert_hint = generate_insert_hint(parallel, insert_mode) -%} {%- set dest_columns = args_dict["dest_columns"] -%} {%- set temp_relation = args_dict["temp_relation"] -%} {%- set target_relation = args_dict["target_relation"] -%} @@ -142,8 +143,7 @@ {% endfor -%} ) {%- else -%} - insert {% if parallel %} /*+parallel({{ parallel }})*/ {% endif %} - {% if insert_mode == 'append' %} /*+ append */ {% endif %} + insert {{ insert_hint }} into {{ target_relation }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} @@ -185,6 +185,7 @@ {% macro oracle__get_incremental_delete_insert_sql(args_dict) %} {%- set parallel = config.get('parallel', none) -%} {%- set insert_mode = config.get('insert_mode', none) -%} + {%- set insert_hint = generate_insert_hint(parallel, insert_mode) -%} {%- set dest_columns = args_dict["dest_columns"] -%} {%- set temp_relation = args_dict["temp_relation"] -%} {%- set target_relation = args_dict["target_relation"] -%} @@ -195,15 +196,13 @@ {%- if unique_key or incremental_predicates -%} BEGIN EXECUTE IMMEDIATE '{{ oracle__get_delete_sql_for_delete_insert_strategy(target_relation, temp_relation, unique_key, incremental_predicates) }}'; - EXECUTE IMMEDIATE 'insert {% if parallel %} /*+parallel({{ parallel }})*/ {% endif %} - {% if insert_mode == 'append' %} /*+ append */ {% endif %} + EXECUTE IMMEDIATE 'INSERT {{ insert_hint }} into {{ target_relation }} ({{ dest_cols_csv }})( select {{ dest_cols_csv }} from {{ temp_relation }})'; END; {%- else -%} - insert {% if parallel %} /*+parallel({{ parallel }})*/ {% endif %} - {% if insert_mode == 'append' %} /*+ append */ {% endif %} + insert {{ insert_hint }} into {{ target_relation }} ({{ dest_cols_csv }}) ( select {{ dest_cols_csv }} diff --git a/requirements.txt b/requirements.txt index 55a934d..e92386f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ dbt-common>=1.1.0,<2.0 dbt-adapters>=1.2.1,<2.0 dbt-core>=1.9.1,<2.0 -oracledb==3.1.0 +oracledb==3.1.1 diff --git a/setup.cfg b/setup.cfg index fe9101b..b838d97 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ install_requires = dbt-common>=1.1.0,<2.0 dbt-adapters>=1.2.1,<2.0 dbt-core~=1.9,<1.10 - oracledb==3.1.0 + oracledb==3.1.1 test_suite=tests test_requires = dbt-tests-adapter~=1.10,<1.11 diff --git a/setup.py b/setup.py index 6b3e75e..8399b9e 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ "dbt-common>=1.1.0,<2.0", "dbt-adapters>=1.2.1,<2.0", "dbt-core~=1.9,<1.10", - "oracledb==3.1.0" + "oracledb==3.1.1" ] test_requirements = [