Feature/campaign performance

.quickstart/quickstart.yml

@@ -4,6 +4,12 @@ schema_key: salesforce_schema
 dbt_versions: ">=1.3.0 <3.0.0"
 
 table_variables:
+  salesforce__campaign_enabled:
+    - Campaign
+  salesforce__campaign_member_enabled:
+    - CampaignMember
+  salesforce__record_type_enabled:
+    - RecordType
   salesforce__user_role_enabled:
     - UserRole
   salesforce__lead_enabled:
@@ -25,6 +31,7 @@ destination_configurations:
       - macro_namespace: dbt_utils
         search_order: [ 'spark_utils', 'dbt_utils' ]
 public_models: [
+  "salesforce__campaign_performance",
   "salesforce__contact_enhanced",
   "salesforce__daily_activity",
   "salesforce__manager_performance",
@@ -40,6 +47,11 @@ supported_vars:
     description: "Additional `ACCOUNT` columns to include in `salesforce__opportunity_enhanced` and `salesforce__contact_enhanced`"
     display_name: "Additional Account Columns"
 
+  salesforce__campaign_pass_through_columns:
+    type: dictionary
+    description: "Additional `CAMPAIGN` columns to include in `salesforce__campaign_performance`"
+    display_name: "Additional Campaign Columns"
+
   salesforce__contact_pass_through_columns:
     type: dictionary
     description: "Additional `CONTACT` columns to include in `salesforce__contact_enhanced`"

CHANGELOG.md

@@ -1,3 +1,22 @@
+# dbt_salesforce v2.3.0
+
+[PR #84](https://github.com/fivetran/dbt_salesforce/pull/84) includes the following updates:
+
+## Schema/Data Change
+**5 total changes • 1 possible breaking change**
+
+| Data Model(s) | Change type | Old | New | Notes |
+| ---------- | ----------- | -------- | -------- | ----- |
+| `salesforce__campaign_performance` | New model | — | One row per campaign | Requires the `campaign` source table. Conditionally joins `campaign_member` and `record_type`. |
+| `salesforce__campaign_daily_history` | New model | — | One row per campaign per day | Disabled by default. Requires Salesforce History Mode and `salesforce__campaign_history_enabled: true`. |
+| `salesforce__opportunity_enhanced` | New column | — | `opportunity_record_type_name` | Conditionally populated when `salesforce__record_type_enabled: true`. Joins `record_type` filtered to `sobject_type = 'Opportunity'`. |
+| `stg_salesforce__campaign` | Column renamed | `record_type_id` | `campaign_member_record_type_id` | Reflects the Salesforce field `CampaignMemberRecordTypeId`, a campaign-level attribute. **Breaking for downstream models that reference `record_type_id` from this staging model.** |
+| `stg_salesforce__campaign_member` | New columns | — | `account_id`, `created_by_id`, `has_opted_out_of_email`, `last_modified_by_id`, `lead_or_contact_id`, `lead_or_contact_owner_id` | Added identity and engagement fields for more complete member-level analysis. |
+
+## Feature Update
+- `salesforce__campaign_performance` includes pipeline totals (`total_pipeline_amount`, `total_won_amount`), member volume (`campaign_member_count`, `responded_count`, `contact_count`, `lead_count`, `opted_out_of_email_count`), and ROI measures (`win_rate`, `cost_per_opportunity`, `cost_per_won_opportunity`, `roi`).
+- Adds support for `salesforce__campaign_pass_through_columns` to pass custom campaign fields through to `salesforce__campaign_performance`. See the [README](https://github.com/fivetran/dbt_salesforce/blob/main/README.md#adding-passthrough-columns) for configuration details.
+
 # dbt_salesforce v2.2.0
 
 [PR #81](https://github.com/fivetran/dbt_salesforce/pull/81) includes the following updates:

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'salesforce'
-version: '2.2.0'
+version: '2.3.0'
 require-dbt-version: [">=1.3.0", "<3.0.0"]
 models:
   salesforce:
@@ -34,6 +34,8 @@ vars:
 
 
     salesforce__account_pass_through_columns: []
+    salesforce__campaign_pass_through_columns: []
+    salesforce__campaign_member_pass_through_columns: []
     salesforce__contact_pass_through_columns: []
     salesforce__event_pass_through_columns: []
     salesforce__lead_pass_through_columns: []

integration_tests/ci/test_scenarios.yml

@@ -19,14 +19,16 @@ test_scenarios:
     vars: {}
     include_incremental: false
 
-  - name: "user role disabled"
+  - name: "features disabled"
     vars:
       salesforce__user_role_enabled: false
+      salesforce__campaign_enabled: false
     include_incremental: false
 
   - name: "history enabled"
     vars:
       salesforce__account_history_enabled: true
+      salesforce__campaign_history_enabled: true
       salesforce__contact_history_enabled: true
       salesforce__opportunity_history_enabled: true
     include_incremental: true

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'salesforce_integration_tests'
-version: '2.2.0'
+version: '2.3.0'
 config-version: 2
 
 profile: 'integration_tests'
@@ -12,6 +12,7 @@ models:
 
 vars:
   # enable history models when generating docs!
+  # salesforce__campaign_history_enabled: true # @docs-include
   # salesforce__contact_history_enabled: true # @docs-include
   # salesforce__account_history_enabled: true # @docs-include
   # salesforce__opportunity_history_enabled: true # @docs-include
@@ -21,7 +22,10 @@ vars:
     salesforce_history_schema: salesforce_integrations_tests_4
 
     salesforce_account_identifier: "sf_account_data"
+    salesforce_campaign_identifier: "sf_campaign_data"
+    salesforce_campaign_member_identifier: "sf_campaign_member_data"
     salesforce_opportunity_identifier: "sf_opportunity_data"
+    salesforce_record_type_identifier: "sf_record_type_data"
     salesforce_user_identifier: "sf_user_data"
     salesforce_user_role_identifier: "sf_user_role_data"
     salesforce_contact_identifier: "sf_contact_data"
@@ -33,6 +37,7 @@ vars:
     salesforce_opportunity_line_item_identifier: "sf_opportunity_line_item_data"
 
     ##history mode identifiers
+    salesforce_campaign_history_identifier: "sf_campaign_history_data"
     salesforce_account_history_identifier: "sf_account_history_data"
     salesforce_contact_history_identifier: "sf_contact_history_data"
     salesforce_opportunity_history_identifier: "sf_opportunity_history_data"
@@ -54,6 +59,36 @@ seeds:
         billing_longitude: float
         shipping_latitude: float
         shipping_longitude: float
+    sf_campaign_data:
+      +column_types:
+        _fivetran_synced: timestamp
+        _fivetran_active: boolean
+        start_date: timestamp
+        end_date: timestamp
+        is_active: boolean
+        is_deleted: boolean
+        budgeted_cost: float
+        actual_cost: float
+        number_of_leads: integer
+        number_of_converted_leads: integer
+        number_of_contacts: integer
+        number_of_responses: integer
+        number_of_opportunities: integer
+        number_of_won_opportunities: integer
+        amount_all_opportunities: float
+    sf_campaign_member_data:
+      +column_types:
+        _fivetran_synced: timestamp
+        _fivetran_active: boolean
+        created_date: timestamp
+        first_responded_date: timestamp
+        has_responded: boolean
+        is_deleted: boolean
+    sf_record_type_data:
+      +column_types:
+        _fivetran_synced: timestamp
+        _fivetran_deleted: boolean
+        is_active: boolean
     sf_opportunity_data:
       +column_types:
         _fivetran_synced: timestamp
@@ -166,6 +201,25 @@ seeds:
         system_modstamp: timestamp
         is_deleted: boolean
 
+    sf_campaign_history_data:
+      +column_types:
+        _fivetran_synced: timestamp
+        _fivetran_active: boolean
+        _fivetran_start: timestamp
+        _fivetran_end: timestamp
+        start_date: timestamp
+        end_date: timestamp
+        is_active: boolean
+        is_deleted: boolean
+        budgeted_cost: float
+        actual_cost: float
+        number_of_leads: integer
+        number_of_converted_leads: integer
+        number_of_contacts: integer
+        number_of_responses: integer
+        number_of_opportunities: integer
+        number_of_won_opportunities: integer
+        amount_all_opportunities: float
     sf_account_history_data:
       +column_types:
         _fivetran_synced: timestamp

integration_tests/seeds/sf_campaign_data.csv

@@ -0,0 +1,5 @@
+_fivetran_synced,_fivetran_active,id,name,type,status,start_date,end_date,is_active,is_deleted,parent_id,budgeted_cost,actual_cost,number_of_leads,number_of_converted_leads,number_of_contacts,number_of_responses,number_of_opportunities,number_of_won_opportunities,amount_all_opportunities,description,campaign_member_record_type_id
+2024-01-15 00:00:00,true,camp001AAA000001,Q1 Webinar Campaign,Webinar,Active,2024-01-01 00:00:00,2024-03-31 00:00:00,true,false,,5000,4200,50,10,30,20,8,3,95000,Q1 webinar campaign for lead generation,rtype001AAA00001
+2024-01-15 00:00:00,true,camp002AAA000002,Email Nurture Series,Email,Completed,2024-02-01 00:00:00,2024-04-30 00:00:00,false,false,,2000,1800,80,15,40,25,5,2,50000,Email nurture campaign,
+2024-01-15 00:00:00,true,camp003AAA000003,Trade Show Spring,Other,Planned,2024-05-01 00:00:00,2024-05-31 00:00:00,true,false,,10000,,20,0,10,5,2,0,0,Spring trade show,rtype002AAA00002
+2024-01-15 00:00:00,true,camp004AAA000004,Social Media Ads,Social,Active,2024-03-01 00:00:00,2024-06-30 00:00:00,true,false,camp001AAA000001,8000,7500,120,25,60,40,12,5,140000,Paid social campaign,

integration_tests/seeds/sf_campaign_history_data.csv

@@ -0,0 +1,5 @@
+_fivetran_active,_fivetran_start,_fivetran_end,_fivetran_synced,id,name,type,status,start_date,end_date,is_active,is_deleted,parent_id,budgeted_cost,actual_cost,number_of_leads,number_of_converted_leads,number_of_contacts,number_of_responses,number_of_opportunities,number_of_won_opportunities,amount_all_opportunities,description,campaign_member_record_type_id
+true,2024-01-01 00:00:00,9999-12-31 23:59:59,2024-01-15 00:00:00,camp001AAA000001,Q1 Webinar Campaign,Webinar,Active,2024-01-01 00:00:00,2024-03-31 00:00:00,true,false,,5000,4200,50,10,30,20,8,3,95000,Q1 webinar campaign for lead generation,rtype001AAA00001
+false,2023-10-01 00:00:00,2023-12-31 23:59:59,2024-01-15 00:00:00,camp001AAA000001,Q1 Webinar Campaign,Webinar,Planned,2024-01-01 00:00:00,2024-03-31 00:00:00,true,false,,5000,0,0,0,0,0,0,0,0,Q1 webinar campaign for lead generation,rtype001AAA00001
+true,2024-02-01 00:00:00,9999-12-31 23:59:59,2024-02-15 00:00:00,camp002AAA000002,Email Nurture Series,Email,Completed,2024-02-01 00:00:00,2024-04-30 00:00:00,false,false,,2000,1800,80,15,40,25,5,2,50000,Email nurture campaign,
+false,2023-11-01 00:00:00,2024-01-31 23:59:59,2024-02-15 00:00:00,camp002AAA000002,Email Nurture Series,Email,Active,2024-02-01 00:00:00,2024-04-30 00:00:00,true,false,,2000,900,40,5,20,10,2,1,25000,Email nurture campaign,

integration_tests/seeds/sf_campaign_member_data.csv

@@ -0,0 +1,6 @@
+_fivetran_synced,_fivetran_active,id,account_id,campaign_id,contact_id,created_by_id,created_date,first_responded_date,has_opted_out_of_email,has_responded,is_deleted,last_modified_by_id,lead_id,lead_or_contact_id,lead_or_contact_owner_id,status
+2024-01-15 00:00:00,true,mem001AAA000001,acct001AAAAAAA,camp001AAA000001,cont001AAAAAAA,user001AAAAAAA,2024-01-10 00:00:00,2024-01-12 00:00:00,false,true,false,user001AAAAAAA,,cont001AAAAAAA,user002AAAAAAA,Responded
+2024-01-15 00:00:00,true,mem002AAA000002,acct002AAAAAAA,camp001AAA000001,,user001AAAAAAA,2024-01-10 00:00:00,,true,false,false,user001AAAAAAA,lead001AAAAAAA,lead001AAAAAAA,user002AAAAAAA,Sent
+2024-01-15 00:00:00,true,mem003AAA000003,acct001AAAAAAA,camp002AAA000002,cont002AAAAAAA,user001AAAAAAA,2024-02-05 00:00:00,,false,false,false,user001AAAAAAA,,cont002AAAAAAA,user002AAAAAAA,Sent
+2024-01-15 00:00:00,true,mem004AAA000004,acct002AAAAAAA,camp002AAA000002,,user001AAAAAAA,2024-02-05 00:00:00,2024-02-10 00:00:00,false,true,false,user001AAAAAAA,lead002AAAAAAA,lead002AAAAAAA,user002AAAAAAA,Responded
+2024-01-15 00:00:00,true,mem005AAA000005,acct001AAAAAAA,camp003AAA000003,cont003AAAAAAA,user001AAAAAAA,2024-05-02 00:00:00,,false,false,false,user001AAAAAAA,,cont003AAAAAAA,user002AAAAAAA,Sent

integration_tests/seeds/sf_record_type_data.csv

@@ -0,0 +1,3 @@
+_fivetran_synced,_fivetran_deleted,id,name,developer_name,sobject_type,is_active,description,namespace_prefix
+2024-01-15 00:00:00,false,rtype001AAA00001,Campaign Standard,Campaign_Standard,Campaign,true,Standard campaign record type,
+2024-01-15 00:00:00,false,rtype002AAA00002,Campaign Premium,Campaign_Premium,Campaign,true,Premium campaign record type,

integration_tests/tests/consistency/consistency_campaign_performance.sql

@@ -0,0 +1,46 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test ensures the campaign_performance end model matches the prior version
+with prod as (
+    select *
+    from {{ target.schema }}_salesforce_prod.salesforce__campaign_performance
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_salesforce_dev.salesforce__campaign_performance
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

integration_tests/tests/consistency/consistency_campaign_performance_count.sql

@@ -0,0 +1,21 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test is to make sure the row counts are the same between versions
+with prod as (
+    select count(*) as prod_rows
+    from {{ target.schema }}_salesforce_prod.salesforce__campaign_performance
+),
+
+dev as (
+    select count(*) as dev_rows
+    from {{ target.schema }}_salesforce_dev.salesforce__campaign_performance
+)
+
+-- test will return values and fail if the row counts don't match
+select *
+from prod
+join dev
+    on prod.prod_rows != dev.dev_rows

integration_tests/tests/integrity/integrity_campaign_performance.sql

@@ -0,0 +1,80 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test validates that the campaign_performance end model correctly reflects
+-- source data: row parity with stg_campaign, member_count from stg_campaign_member,
+-- and total_won_amount from stg_opportunity
+
+with end_model as (
+    select *
+    from {{ target.schema }}_salesforce_dev.salesforce__campaign_performance
+),
+
+-- verify 1:1 relationship with staging campaign -- no fanout, no missing rows
+campaign_row_parity as (
+    select count(*) as end_model_count
+    from end_model
+),
+
+stg_campaign_count as (
+    select count(*) as stg_count
+    from {{ target.schema }}_salesforce_dev.stg_salesforce__campaign
+),
+
+row_parity_check as (
+    select *
+    from campaign_row_parity
+    join stg_campaign_count
+        on end_model_count != stg_count
+),
+
+-- verify member_count matches direct count from stg_campaign_member
+member_count_check as (
+    select
+        e.campaign_id,
+        e.member_count as end_model_member_count,
+        coalesce(m.stg_member_count, 0) as stg_member_count
+    from end_model e
+    left join (
+        select
+            campaign_id,
+            count(*) as stg_member_count
+        from {{ target.schema }}_salesforce_dev.stg_salesforce__campaign_member
+        group by 1
+    ) m
+        on e.campaign_id = m.campaign_id
+    where coalesce(e.member_count, 0) != coalesce(m.stg_member_count, 0)
+),
+
+-- verify total_won_amount matches direct sum from stg_opportunity
+won_amount_check as (
+    select
+        e.campaign_id,
+        e.total_won_amount as end_model_won_amount,
+        o.stg_won_amount
+    from end_model e
+    left join (
+        select
+            campaign_id,
+            sum(amount) as stg_won_amount
+        from {{ target.schema }}_salesforce_dev.stg_salesforce__opportunity
+        where is_won = true
+            and campaign_id is not null
+        group by 1
+    ) o
+        on e.campaign_id = o.campaign_id
+    where coalesce(e.total_won_amount, 0) != coalesce(o.stg_won_amount, 0)
+),
+
+final as (
+    select campaign_id, 'member_count mismatch' as reason from member_count_check
+    union all
+    select campaign_id, 'total_won_amount mismatch' as reason from won_amount_check
+    union all
+    select cast(null as string), 'campaign row count mismatch' as reason from row_parity_check
+)
+
+select *
+from final

macros/staging/get_campaign_columns.sql

@@ -0,0 +1,35 @@
+
+{% macro get_campaign_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_active", "datatype": "boolean"},
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "actual_cost", "datatype": dbt.type_numeric()},
+    {"name": "amount_all_opportunities", "datatype": dbt.type_numeric()},
+    {"name": "budgeted_cost", "datatype": dbt.type_numeric()},
+    {"name": "description", "datatype": dbt.type_string()},
+    {"name": "end_date", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_string()},
+    {"name": "is_active", "datatype": "boolean"},
+    {"name": "is_deleted", "datatype": "boolean"},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "number_of_contacts", "datatype": dbt.type_int()},
+    {"name": "number_of_converted_leads", "datatype": dbt.type_int()},
+    {"name": "number_of_leads", "datatype": dbt.type_int()},
+    {"name": "number_of_opportunities", "datatype": dbt.type_int()},
+    {"name": "number_of_responses", "datatype": dbt.type_int()},
+    {"name": "number_of_won_opportunities", "datatype": dbt.type_int()},
+    {"name": "campaign_member_record_type_id", "datatype": dbt.type_string()},
+    {"name": "parent_id", "datatype": dbt.type_string()},
+    {"name": "start_date", "datatype": dbt.type_timestamp()},
+    {"name": "status", "datatype": dbt.type_string()},
+    {"name": "type", "datatype": dbt.type_string()}
+] %}
+
+{{ salesforce.add_renamed_columns(columns) }}
+
+{{ fivetran_utils.add_pass_through_columns(columns, var('salesforce__campaign_pass_through_columns')) }}
+
+{{ return(columns) }}
+
+{% endmacro %}