Pull country from metrics in quick_suggest ping

CODEOWNERS

@@ -22,7 +22,7 @@ dags.yaml
 /sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/ @mozilla/revenue_forecasting_data_reviewers
 /sql/moz-fx-data-shared-prod/search_derived/search_dau_aggregates_v1/ @mozilla/revenue_forecasting_data_reviewers
 # Contextual Services
-/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2 @mozilla/request_payload_reviewers
+/sql/**/request_payload_suggest_v2 @mozilla/request_payload_reviewers
 /sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2 @mozilla/request_payload_reviewers
 /sql/moz-fx-data-shared-prod/contextual_services_derived/suggest_revenue_levers_daily_v1 @mozilla/revenue_forecasting_data_reviewers
 /sql/moz-fx-data-shared-prod/monitoring_derived/jobs_by_organization_v1 @mozilla/dataops

sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql

@@ -14,7 +14,10 @@ combined AS (
     metrics.uuid.quick_suggest_context_id AS context_id,
     DATE(submission_timestamp) AS submission_date,
     'desktop' AS form_factor,
-    normalized_country_code AS country,
+    -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now
+    -- receives geo information from the client rather than from Glean ingestion's
+    -- IP geolocation. We no longer send subdivision, only country.
+    COALESCE(normalized_country_code, metrics.string.quick_suggest_country) AS country,
     LOWER(metrics.string.quick_suggest_advertiser) AS advertiser,
     SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os,
     client_info.app_channel AS release_channel,
@@ -102,7 +105,8 @@ combined AS (
     metrics.uuid.fx_suggest_context_id AS context_id,
     DATE(submission_timestamp) AS submission_date,
     'phone' AS form_factor,
-    normalized_country_code AS country,
+    -- With shift to OHTTP, we expect to stop receiving normalized_country_code soon
+    COALESCE(normalized_country_code, metrics.string.fx_suggest_country) AS country,
     metrics.string.fx_suggest_advertiser AS advertiser,
     SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os,
     client_info.app_channel AS release_channel,

sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql

@@ -20,7 +20,10 @@ combined AS (
       "impression"
     ) AS event_type,
     'desktop' AS form_factor,
-    normalized_country_code AS country,
+    -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now
+    -- receives geo information from the client rather than from Glean ingestion's
+    -- IP geolocation. We no longer send subdivision, only country.
+    COALESCE(normalized_country_code, metrics.string.quick_suggest_country) AS country,
     metadata.geo.subdivision1 AS subdivision1,
     metrics.string.quick_suggest_advertiser AS advertiser,
     client_info.app_channel AS release_channel,
@@ -117,7 +120,8 @@ combined AS (
       "impression"
     ) AS event_type,
     'phone' AS form_factor,
-    normalized_country_code AS country,
+    -- With shift to OHTTP, we expect to stop receiving normalized_country_code soon
+    COALESCE(normalized_country_code, metrics.string.fx_suggest_country) AS country,
     metadata.geo.subdivision1 AS subdivision1,
     metrics.string.fx_suggest_advertiser AS advertiser,
     client_info.app_channel AS release_channel,

sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql

@@ -18,7 +18,10 @@ ping_data AS (
       "click",
       "impression"
     ) AS interaction_type,
-    metadata.geo.country AS country_code,
+    -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now
+    -- receives geo information from the client rather than from Glean ingestion's
+    -- IP geolocation. We no longer send subdivision, only country.
+    COALESCE(metadata.geo.country, metrics.string.quick_suggest_country) AS country,
     metadata.geo.subdivision1 AS region_code,
     metadata.user_agent.os AS os_family,
     metadata.user_agent.version AS product_version,

sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql

@@ -45,7 +45,10 @@ WITH impressions AS (
     sample_id,
     metrics.boolean.quick_suggest_is_clicked AS is_clicked,
     client_info.locale AS locale,
-    metadata.geo.country,
+    -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now
+    -- receives geo information from the client rather than from Glean ingestion's
+    -- IP geolocation. We no longer send subdivision, only country.
+    COALESCE(metadata.geo.country, metrics.string.quick_suggest_country) AS country,
     metadata.geo.subdivision1 AS region,
     normalized_os,
     normalized_os_version,

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml

@@ -3,10 +3,9 @@
   submission_date: "2030-01-01"
   source: suggest
   provider: merino
-  event_type: impression
+  event_type: click
   form_factor: desktop
-  country: US
-  subdivision1: AZ
+  country: DE
   advertiser: ad1
   release_channel: release
   match_type: "firefox-suggest"
@@ -17,7 +16,12 @@
   user_count: 1
   query_type: branded
 - <<: *suggest_base
-  event_type: click
+  event_type: impression
+  country: US
+  subdivision1: AZ
+- <<: *suggest_base
+  country: US
+  subdivision1: AZ
 - &suggest_mobile
   submission_date: "2030-01-01"
   source: suggest
@@ -36,6 +40,20 @@
   query_type: branded
 - <<: *suggest_mobile
   normalized_os: iOS
+- submission_date: "2030-01-01"
+  source: suggest
+  provider: remote settings
+  event_type: impression
+  form_factor: phone
+  country: VN
+  advertiser: ad4
+  release_channel: release
+  match_type: "firefox-suggest"
+  normalized_os: Android
+  position: 1
+  event_count: 1
+  user_count: 1
+  query_type: branded
 - &topsites_base
   submission_date: "2030-01-01"
   source: topsites

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.fenix.fx_suggest.yaml

@@ -8,6 +8,7 @@
     string:
       fx_suggest_advertiser: ad4
       fx_suggest_ping_type: fxsuggest-impression
+      fx_suggest_country: null
     uuid:
       fx_suggest_context_id: fenix-b
   metadata:
@@ -18,6 +19,19 @@
   client_info:
     app_channel: release
   normalized_country_code: US
+- <<: *suggest_base
+  metrics:
+    <<: *suggest_metrics
+    string:
+      fx_suggest_advertiser: ad4
+      fx_suggest_ping_type: fxsuggest-impression
+      fx_suggest_country: VN
+  metadata:
+    geo:
+      subdivision1: null
+    user_agent:
+      os: Android
+  normalized_country_code: null
 - <<: *suggest_base
   metrics:
     <<: *suggest_metrics

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml

@@ -28,3 +28,15 @@
     string:
       <<: *click_strings
       quick_suggest_ping_type: quicksuggest-impression
+- <<: *click
+  metadata:
+    geo:
+      subdivision1: null
+    user_agent:
+      os: "Windows 11"
+  metrics:
+    <<: *click_metrics
+    string:
+      <<: *click_strings
+      quick_suggest_country: DE
+  normalized_country_code: null

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.fenix.fx_suggest.yaml

@@ -8,6 +8,7 @@
     string:
       fx_suggest_advertiser: ad4
       fx_suggest_ping_type: fxsuggest-click
+      fx_suggest_country: null
     uuid:
       fx_suggest_context_id: fenix-b
   metadata:

tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml

@@ -8,6 +8,7 @@
       subdivision1: AZ
     user_agent:
       os: "Windows 11"
+      version: 130
   metrics: &click_metrics
     boolean:
       quick_suggest_improve_suggest_experience: true
@@ -19,6 +20,7 @@
       quick_suggest_match_type: firefox-suggest
       quick_suggest_ping_type: quicksuggest-click
       quick_suggest_request_id: HASH123
+      quick_suggest_country: null
     uuid:
       quick_suggest_context_id: a
   normalized_country_code: US

tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/expect.yaml

@@ -116,3 +116,24 @@
   reporting_url: https://example.com/duff-beer
   submission_timestamp: "2021-01-01T13:03:00+00:00"
   version: "134"
+- advertiser: omni consumer products
+  block_id: 205
+  browser: firefox(141)
+  context_id: cid15
+  country: US
+  form_factor: desktop
+  is_clicked: false
+  locale: en-US
+  normalized_channel: release
+  normalized_os: windows
+  normalized_os_version: 15.2
+  os_family: windows
+  position: 15
+  query: robot police
+  reporting_url: https://example.com/ocp
+  request_id: rid15
+  sequence_no: 15
+  session_id: sid15
+  submission_timestamp: "2021-01-01T13:05:00+00:00"
+  timestamp: "2021-01-01T13:05:00+00:00"
+  version: "141"

tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml

@@ -102,3 +102,30 @@
   normalized_os: Windows
   normalized_os_version: "11.0"
   sample_id: null
+- submission_timestamp: "2021-01-01 13:05:00"
+  client_info:
+    app_display_version: "141.0.1"
+    locale: en-US
+  metadata:
+    geo:
+      country: null
+      subdivision1: null
+  metrics:
+    boolean:
+      quick_suggest_is_clicked: false
+    quantity:
+      quick_suggest_position: 15
+    string:
+      quick_suggest_advertiser: omni consumer products
+      quick_suggest_block_id: 205
+      quick_suggest_ping_type: "quicksuggest-impression"
+      quick_suggest_request_id: rid15
+      quick_suggest_country: US
+    url2:
+      quick_suggest_reporting_url: "https://example.com/ocp"
+    uuid:
+      quick_suggest_context_id: cid15
+  normalized_channel: release
+  normalized_os: windows
+  normalized_os_version: "15.2"
+  sample_id: null

tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.search_terms_derived.merino_log_sanitized_v3.yaml

@@ -64,3 +64,14 @@
   form_factor: desktop
   browser: firefox(134)
   os_family: windows
+- timestamp: "2021-01-01 13:05:00.000000"
+  request_id: rid15
+  session_id: sid15
+  sequence_no: 15
+  query: "robot police"
+  country: null
+  region: null
+  dma: null
+  form_factor: desktop
+  browser: firefox(141)
+  os_family: windows