diff --git a/CODEOWNERS b/CODEOWNERS index b9e270da411..7d830d39957 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -22,7 +22,7 @@ dags.yaml /sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/ @mozilla/revenue_forecasting_data_reviewers /sql/moz-fx-data-shared-prod/search_derived/search_dau_aggregates_v1/ @mozilla/revenue_forecasting_data_reviewers # Contextual Services -/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2 @mozilla/request_payload_reviewers +/sql/**/request_payload_suggest_v2 @mozilla/request_payload_reviewers /sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2 @mozilla/request_payload_reviewers /sql/moz-fx-data-shared-prod/contextual_services_derived/suggest_revenue_levers_daily_v1 @mozilla/revenue_forecasting_data_reviewers /sql/moz-fx-data-shared-prod/monitoring_derived/jobs_by_organization_v1 @mozilla/dataops diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql index eb0ef23ce04..6b686b1f03f 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql @@ -14,7 +14,10 @@ combined AS ( metrics.uuid.quick_suggest_context_id AS context_id, DATE(submission_timestamp) AS submission_date, 'desktop' AS form_factor, - normalized_country_code AS country, + -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now + -- receives geo information from the client rather than from Glean ingestion's + -- IP geolocation. We no longer send subdivision, only country. + COALESCE(normalized_country_code, metrics.string.quick_suggest_country) AS country, LOWER(metrics.string.quick_suggest_advertiser) AS advertiser, SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os, client_info.app_channel AS release_channel, @@ -102,7 +105,8 @@ combined AS ( metrics.uuid.fx_suggest_context_id AS context_id, DATE(submission_timestamp) AS submission_date, 'phone' AS form_factor, - normalized_country_code AS country, + -- With shift to OHTTP, we expect to stop receiving normalized_country_code soon + COALESCE(normalized_country_code, metrics.string.fx_suggest_country) AS country, metrics.string.fx_suggest_advertiser AS advertiser, SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os, client_info.app_channel AS release_channel, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql index c647feb42e1..50c63a432ac 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql @@ -20,7 +20,10 @@ combined AS ( "impression" ) AS event_type, 'desktop' AS form_factor, - normalized_country_code AS country, + -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now + -- receives geo information from the client rather than from Glean ingestion's + -- IP geolocation. We no longer send subdivision, only country. + COALESCE(normalized_country_code, metrics.string.quick_suggest_country) AS country, metadata.geo.subdivision1 AS subdivision1, metrics.string.quick_suggest_advertiser AS advertiser, client_info.app_channel AS release_channel, @@ -117,7 +120,8 @@ combined AS ( "impression" ) AS event_type, 'phone' AS form_factor, - normalized_country_code AS country, + -- With shift to OHTTP, we expect to stop receiving normalized_country_code soon + COALESCE(normalized_country_code, metrics.string.fx_suggest_country) AS country, metadata.geo.subdivision1 AS subdivision1, metrics.string.fx_suggest_advertiser AS advertiser, client_info.app_channel AS release_channel, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql index 20dd0eb16e2..eeae79f2e21 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql @@ -18,7 +18,10 @@ ping_data AS ( "click", "impression" ) AS interaction_type, - metadata.geo.country AS country_code, + -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now + -- receives geo information from the client rather than from Glean ingestion's + -- IP geolocation. We no longer send subdivision, only country. + COALESCE(metadata.geo.country, metrics.string.quick_suggest_country) AS country, metadata.geo.subdivision1 AS region_code, metadata.user_agent.os AS os_family, metadata.user_agent.version AS product_version, diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml index 0a0e3cda839..d8b41b82473 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml @@ -3,10 +3,9 @@ submission_date: "2030-01-01" source: suggest provider: merino - event_type: impression + event_type: click form_factor: desktop - country: US - subdivision1: AZ + country: DE advertiser: ad1 release_channel: release match_type: "firefox-suggest" @@ -17,7 +16,12 @@ user_count: 1 query_type: branded - <<: *suggest_base - event_type: click + event_type: impression + country: US + subdivision1: AZ +- <<: *suggest_base + country: US + subdivision1: AZ - &suggest_mobile submission_date: "2030-01-01" source: suggest @@ -36,6 +40,20 @@ query_type: branded - <<: *suggest_mobile normalized_os: iOS +- submission_date: "2030-01-01" + source: suggest + provider: remote settings + event_type: impression + form_factor: phone + country: VN + advertiser: ad4 + release_channel: release + match_type: "firefox-suggest" + normalized_os: Android + position: 1 + event_count: 1 + user_count: 1 + query_type: branded - &topsites_base submission_date: "2030-01-01" source: topsites diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.fenix.fx_suggest.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.fenix.fx_suggest.yaml index 96f04b1f531..58ffa2630d4 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.fenix.fx_suggest.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.fenix.fx_suggest.yaml @@ -8,6 +8,7 @@ string: fx_suggest_advertiser: ad4 fx_suggest_ping_type: fxsuggest-impression + fx_suggest_country: null uuid: fx_suggest_context_id: fenix-b metadata: @@ -18,6 +19,19 @@ client_info: app_channel: release normalized_country_code: US +- <<: *suggest_base + metrics: + <<: *suggest_metrics + string: + fx_suggest_advertiser: ad4 + fx_suggest_ping_type: fxsuggest-impression + fx_suggest_country: VN + metadata: + geo: + subdivision1: null + user_agent: + os: Android + normalized_country_code: null - <<: *suggest_base metrics: <<: *suggest_metrics diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml index a43b7ec3802..2469bca6e5d 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml @@ -28,3 +28,15 @@ string: <<: *click_strings quick_suggest_ping_type: quicksuggest-impression +- <<: *click + metadata: + geo: + subdivision1: null + user_agent: + os: "Windows 11" + metrics: + <<: *click_metrics + string: + <<: *click_strings + quick_suggest_country: DE + normalized_country_code: null diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.fenix.fx_suggest.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.fenix.fx_suggest.yaml index 1e68a463af1..28978c36cde 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.fenix.fx_suggest.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.fenix.fx_suggest.yaml @@ -8,6 +8,7 @@ string: fx_suggest_advertiser: ad4 fx_suggest_ping_type: fxsuggest-click + fx_suggest_country: null uuid: fx_suggest_context_id: fenix-b metadata: diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml index 4ce20e7eed5..2e0cd766d8a 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml @@ -8,6 +8,7 @@ subdivision1: AZ user_agent: os: "Windows 11" + version: 130 metrics: &click_metrics boolean: quick_suggest_improve_suggest_experience: true @@ -19,6 +20,7 @@ quick_suggest_match_type: firefox-suggest quick_suggest_ping_type: quicksuggest-click quick_suggest_request_id: HASH123 + quick_suggest_country: null uuid: quick_suggest_context_id: a normalized_country_code: US