From 0e3be9d52bbbefe21e5899d9fb5af1d12b9898f4 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 12 May 2026 20:39:30 -0500 Subject: [PATCH 1/3] Buildkite improvements --- .buildkite/hooks/pre-command | 25 -------- .buildkite/pipeline.yml | 74 ------------------------ .buildkite/scripts/run_models.sh | 29 ---------- integration_tests/ci/sample.profiles.yml | 50 ---------------- integration_tests/ci/test_scenarios.yml | 37 ++++++++++++ integration_tests/dbt_project.yml | 34 ++++++++--- 6 files changed, 64 insertions(+), 185 deletions(-) delete mode 100644 .buildkite/hooks/pre-command delete mode 100644 .buildkite/pipeline.yml delete mode 100644 .buildkite/scripts/run_models.sh delete mode 100644 integration_tests/ci/sample.profiles.yml create mode 100644 integration_tests/ci/test_scenarios.yml diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command deleted file mode 100644 index 81b7b002..00000000 --- a/.buildkite/hooks/pre-command +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -e - -# Export secrets for Docker containers. -# Restrict exposing secrets only to the steps that need them -export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 26036027..00000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,74 +0,0 @@ -steps: - - label: ":postgres: Run Tests - Postgres" - key: "run-dbt-postgres" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_POSTGRES_DBT_DBNAME" - - "CI_POSTGRES_DBT_HOST" - - "CI_POSTGRES_DBT_PASS" - - "CI_POSTGRES_DBT_USER" - commands: | - bash .buildkite/scripts/run_models.sh postgres - - - label: ":snowflake-db: Run Tests - Snowflake" - key: "run_dbt_snowflake" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_SNOWFLAKE_DBT_ACCOUNT" - - "CI_SNOWFLAKE_DBT_DATABASE" - - "CI_SNOWFLAKE_DBT_PASS" - - "CI_SNOWFLAKE_DBT_ROLE" - - "CI_SNOWFLAKE_DBT_USER" - - "CI_SNOWFLAKE_DBT_WAREHOUSE" - commands: | - bash .buildkite/scripts/run_models.sh snowflake - - - label: ":gcloud: Run Tests - BigQuery" - key: "run_dbt_bigquery" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "GCLOUD_SERVICE_KEY" - commands: | - bash .buildkite/scripts/run_models.sh bigquery - - - label: ":amazon-redshift: Run Tests - Redshift" - key: "run_dbt_redshift" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_REDSHIFT_DBT_DBNAME" - - "CI_REDSHIFT_DBT_HOST" - - "CI_REDSHIFT_DBT_PASS" - - "CI_REDSHIFT_DBT_USER" - commands: | - bash .buildkite/scripts/run_models.sh redshift - - - label: ":databricks: Run Tests - Databricks" - key: "run_dbt_databricks" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_DATABRICKS_DBT_HOST" - - "CI_DATABRICKS_DBT_HTTP_PATH" - - "CI_DATABRICKS_DBT_TOKEN" - - "CI_DATABRICKS_DBT_CATALOG" - commands: | - bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh deleted file mode 100644 index a14414f5..00000000 --- a/.buildkite/scripts/run_models.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -apt-get update -apt-get install libsasl2-dev - -python3 -m venv venv -. venv/bin/activate -pip install --upgrade pip setuptools -pip install -r integration_tests/requirements.txt -mkdir -p ~/.dbt -cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml - -db=$1 -echo `pwd` -cd integration_tests -dbt deps -dbt seed --target "$db" --full-refresh -dbt source freshness --target "$db" || echo "...Only verifying freshness runs..." -dbt run --target "$db" --full-refresh -dbt test --target "$db" -dbt run --vars '{salesforce__user_role_enabled: false}' --target "$db" --full-refresh -dbt test --target "$db" -dbt run --vars '{salesforce__account_history_enabled: true, salesforce__contact_history_enabled: true, salesforce__opportunity_history_enabled: true}' --target "$db" --full-refresh -dbt test --target "$db" -dbt run --vars '{salesforce__account_history_enabled: true, salesforce__contact_history_enabled: true, salesforce__opportunity_history_enabled: true}' --target "$db" -dbt test --target "$db" -dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml deleted file mode 100644 index d2da4fdb..00000000 --- a/integration_tests/ci/sample.profiles.yml +++ /dev/null @@ -1,50 +0,0 @@ - -# HEY! This file is used in the dbt package integrations tests with Buildkite. -# You should __NEVER__ check credentials into version control. Thanks for reading :) - -integration_tests: - target: redshift - outputs: - redshift: - type: redshift - host: "{{ env_var('CI_REDSHIFT_DBT_HOST') }}" - user: "{{ env_var('CI_REDSHIFT_DBT_USER') }}" - pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" - dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" - port: 5439 - schema: salesforce_integrations_tests_4 - threads: 8 - bigquery: - type: bigquery - method: service-account-json - project: 'dbt-package-testing' - schema: salesforce_integrations_tests_4 - threads: 8 - keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" - snowflake: - type: snowflake - account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}" - user: "{{ env_var('CI_SNOWFLAKE_DBT_USER') }}" - password: "{{ env_var('CI_SNOWFLAKE_DBT_PASS') }}" - role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" - database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" - warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: salesforce_integrations_tests_4 - threads: 8 - postgres: - type: postgres - host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}" - user: "{{ env_var('CI_POSTGRES_DBT_USER') }}" - pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" - dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" - port: 5432 - schema: salesforce_integrations_tests_4 - threads: 8 - databricks: - catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" - host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" - http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: salesforce_integrations_tests_4 - threads: 8 - token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" - type: databricks \ No newline at end of file diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml new file mode 100644 index 00000000..51894eba --- /dev/null +++ b/integration_tests/ci/test_scenarios.yml @@ -0,0 +1,37 @@ +# Only set if not using default adapter ranges +# dbt_adapter_versions: +# snowflake: ">=1.3.0,<2.0.0" +# bigquery: ">=1.3.0,<2.0.0" +# postgres: ">=1.3.0,<2.0.0" +# redshift: ">=1.3.0,<2.0.0" +# databricks: ">=1.6.0,<2.0.0" + +schema_variable_name: "salesforce_schema" + +include_databricks_sql: false +include_sqlserver: false +include_dbt_compile: true + +test_scenarios: + - name: "Default" + vars: {} + include_incremental: false + + - name: "user role disabled" + vars: + salesforce__user_role_enabled: false + include_incremental: false + + - name: "history enabled" + vars: + salesforce__account_history_enabled: true + salesforce__contact_history_enabled: true + salesforce__opportunity_history_enabled: true + include_incremental: false + + - name: "history enabled incremental" + vars: + salesforce__account_history_enabled: true + salesforce__contact_history_enabled: true + salesforce__opportunity_history_enabled: true + include_incremental: true \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 6bb35d95..5fa5c4ea 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -42,11 +42,10 @@ seeds: show: False salesforce_integration_tests: +quote_columns: "{{ true if target.type in ('redshift', 'postgres') else false }}" - +column_types: - _fivetran_synced: timestamp - _fivetran_active: boolean sf_account_data: +column_types: + _fivetran_synced: timestamp + _fivetran_active: boolean last_activity_date: timestamp last_referenced_date: timestamp last_viewed_date: timestamp @@ -57,6 +56,8 @@ seeds: shipping_longitude: float sf_opportunity_data: +column_types: + _fivetran_synced: timestamp + _fivetran_active: boolean close_date: timestamp created_date: timestamp last_activity_date: timestamp @@ -66,6 +67,8 @@ seeds: probability: float sf_user_data: +column_types: + _fivetran_synced: timestamp + _fivetran_active: boolean last_login_date: timestamp last_referenced_date: timestamp last_viewed_date: timestamp @@ -74,7 +77,8 @@ seeds: longitude: float sf_contact_data: +column_types: - birthdate: timestamp + _fivetran_synced: timestamp + _fivetran_active: boolean created_date: timestamp email_bounced_date: timestamp last_activity_date: timestamp @@ -87,6 +91,8 @@ seeds: is_deleted: boolean sf_lead_data: +column_types: + _fivetran_synced: timestamp + _fivetran_active: boolean converted_date: timestamp created_date: timestamp email_bounced_date: timestamp @@ -98,6 +104,8 @@ seeds: is_deleted: boolean sf_task_data: +column_types: + _fivetran_synced: timestamp + _fivetran_active: boolean activity_date: timestamp completed_date_time: timestamp created_date: timestamp @@ -109,10 +117,11 @@ seeds: is_deleted: boolean sf_event_data: +column_types: + _fivetran_synced: timestamp + _fivetran_active: boolean activity_date: timestamp activity_date_time: timestamp created_date: timestamp - end_date: timestamp end_date_time: timestamp last_modified_date : timestamp recurrence_2_pattern_start_date : timestamp @@ -124,6 +133,8 @@ seeds: is_deleted: boolean sf_product_2_data: +column_types: + _fivetran_synced: timestamp + _fivetran_active: boolean created_date: timestamp last_modified_date: timestamp last_referenced_date: timestamp @@ -132,6 +143,8 @@ seeds: is_deleted: boolean sf_order_data: +column_types: + _fivetran_synced: timestamp + _fivetran_active: boolean activated_date: timestamp created_date: timestamp effective_date: timestamp @@ -143,6 +156,8 @@ seeds: is_deleted: boolean sf_opportunity_line_item_data: +column_types: + _fivetran_synced: timestamp + _fivetran_active: boolean created_date: timestamp last_modified_date: timestamp last_referenced_date: timestamp @@ -153,6 +168,8 @@ seeds: sf_account_history_data: +column_types: + _fivetran_synced: timestamp + _fivetran_active: boolean annual_revenue: float last_activity_date: timestamp last_referenced_date: timestamp @@ -160,13 +177,16 @@ seeds: is_deleted: boolean sf_contact_history_data: +column_types: - birthdate: timestamp + _fivetran_synced: timestamp + _fivetran_active: boolean last_activity_date: timestamp last_modified_date: timestamp last_viewed_date: timestamp is_deleted: boolean sf_opportunity_history_data: +column_types: + _fivetran_synced: timestamp + _fivetran_active: boolean close_date: timestamp created_date: timestamp last_activity_date: timestamp @@ -189,4 +209,4 @@ flags: send_anonymous_usage_stats: False - use_colors: True + use_colors: True \ No newline at end of file From 3e230402e84299de1e888423d8c6fe33c0699a15 Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 12 May 2026 22:09:32 -0500 Subject: [PATCH 2/3] Update test_scenarios.yml --- integration_tests/ci/test_scenarios.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index 51894eba..a3cb9d4a 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -27,11 +27,4 @@ test_scenarios: salesforce__account_history_enabled: true salesforce__contact_history_enabled: true salesforce__opportunity_history_enabled: true - include_incremental: false - - - name: "history enabled incremental" - vars: - salesforce__account_history_enabled: true - salesforce__contact_history_enabled: true - salesforce__opportunity_history_enabled: true - include_incremental: true \ No newline at end of file + include_incremental: true From 19762aba531624ace944ad90421994899f619f3f Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 12 May 2026 22:25:16 -0500 Subject: [PATCH 3/3] Add additional schema variable for salesforce history --- integration_tests/ci/test_scenarios.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index a3cb9d4a..21377b09 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -7,6 +7,8 @@ # databricks: ">=1.6.0,<2.0.0" schema_variable_name: "salesforce_schema" +additional_schema_variables: + - "salesforce_history_schema" include_databricks_sql: false include_sqlserver: false