From 1628fb04b949b94a07dda3149795b90cc221b5f7 Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Fri, 23 May 2025 18:14:29 -0400 Subject: [PATCH 01/19] Pull country from metrics in quick_suggest ping --- .../event_aggregates_suggest_v1/query.sql | 9 ++++++++- .../event_aggregates_v1/query.sql | 15 +++++++++++++-- .../request_payload_suggest_v2/query.sql | 15 +++++++++++++-- .../suggest_impression_sanitized_v3/query.sql | 12 ++++++++++-- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql index eb0ef23ce04..78681233008 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql @@ -14,7 +14,14 @@ combined AS ( metrics.uuid.quick_suggest_context_id AS context_id, DATE(submission_timestamp) AS submission_date, 'desktop' AS form_factor, - normalized_country_code AS country, + -- As of Firefox 140, the quick_suggest ping is sent via OHTTP and now + -- receives geo information from the client rather than from Glean ingestion's + -- IP geolocation. We no longer send subdivision, only country. + IF( + SAFE_CAST(metadata.user_agent.version AS INT64) < 140, + normalized_country_code, + metrics.string.country, + ) AS country, LOWER(metrics.string.quick_suggest_advertiser) AS advertiser, SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os, client_info.app_channel AS release_channel, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql index 563424dd7e8..489f7d23bab 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql @@ -20,8 +20,19 @@ combined AS ( "impression" ) AS event_type, 'desktop' AS form_factor, - normalized_country_code AS country, - metadata.geo.subdivision1 AS subdivision1, + -- As of Firefox 140, the quick_suggest ping is sent via OHTTP and now + -- receives geo information from the client rather than from Glean ingestion's + -- IP geolocation. We no longer send subdivision, only country. + IF( + SAFE_CAST(metadata.user_agent.version AS INT64) < 140, + normalized_country_code, + metrics.string.country, + ) AS country, + IF( + SAFE_CAST(metadata.user_agent.version AS INT64) < 140, + metadata.geo.subdivision1, + CAST(NULL AS STRING) + ) AS subdivision1, metrics.string.quick_suggest_advertiser AS advertiser, client_info.app_channel AS release_channel, metrics.quantity.quick_suggest_position AS position, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql index 20dd0eb16e2..186ae941e3d 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql @@ -18,8 +18,19 @@ ping_data AS ( "click", "impression" ) AS interaction_type, - metadata.geo.country AS country_code, - metadata.geo.subdivision1 AS region_code, + -- As of Firefox 140, the quick_suggest ping is sent via OHTTP and now + -- receives geo information from the client rather than from Glean ingestion's + -- IP geolocation. We no longer send subdivision, only country. + IF( + SAFE_CAST(metadata.user_agent.version AS INT64) < 140, + metadata.geo.country, + metrics.string.country, + ) AS country_code, + IF( + SAFE_CAST(metadata.user_agent.version AS INT64) < 140, + metadata.geo.subdivision1, + CAST(NULL AS STRING) + ) AS region_code, metadata.user_agent.os AS os_family, metadata.user_agent.version AS product_version, FROM diff --git a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql index 87a0759717d..f981d4a3eae 100644 --- a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql +++ b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql @@ -45,8 +45,16 @@ WITH impressions AS ( sample_id, metrics.boolean.quick_suggest_is_clicked AS is_clicked, client_info.locale AS locale, - metadata.geo.country, - metadata.geo.subdivision1 AS region, + IF( + SAFE_CAST(metadata.user_agent.version AS INT64) < 140, + metadata.geo.country, + metrics.string.country, + ) AS country, + IF( + SAFE_CAST(metadata.user_agent.version AS INT64) < 140, + metadata.geo.subdivision1, + CAST(NULL AS STRING) + ) AS region, normalized_os, normalized_os_version, normalized_channel, From 68e8fc9480039299dfbb1bc5d17c52c71237a2d0 Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Tue, 1 Jul 2025 18:43:41 -0400 Subject: [PATCH 02/19] Fix queries --- .../event_aggregates_suggest_v1/query.sql | 2 +- .../contextual_services_derived/event_aggregates_v1/query.sql | 2 +- .../request_payload_suggest_v2/query.sql | 2 +- .../suggest_impression_sanitized_v3/query.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql index 78681233008..264ede83fb1 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql @@ -20,7 +20,7 @@ combined AS ( IF( SAFE_CAST(metadata.user_agent.version AS INT64) < 140, normalized_country_code, - metrics.string.country, + metrics.string.quick_suggest_country ) AS country, LOWER(metrics.string.quick_suggest_advertiser) AS advertiser, SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql index 489f7d23bab..d15eb3cbb41 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql @@ -26,7 +26,7 @@ combined AS ( IF( SAFE_CAST(metadata.user_agent.version AS INT64) < 140, normalized_country_code, - metrics.string.country, + metrics.string.quick_suggest_country ) AS country, IF( SAFE_CAST(metadata.user_agent.version AS INT64) < 140, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql index 186ae941e3d..69e2e2ad9c8 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql @@ -24,7 +24,7 @@ ping_data AS ( IF( SAFE_CAST(metadata.user_agent.version AS INT64) < 140, metadata.geo.country, - metrics.string.country, + metrics.string.quick_suggest_country ) AS country_code, IF( SAFE_CAST(metadata.user_agent.version AS INT64) < 140, diff --git a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql index f981d4a3eae..9eaf2649118 100644 --- a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql +++ b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql @@ -48,7 +48,7 @@ WITH impressions AS ( IF( SAFE_CAST(metadata.user_agent.version AS INT64) < 140, metadata.geo.country, - metrics.string.country, + metrics.string.quick_suggest_country ) AS country, IF( SAFE_CAST(metadata.user_agent.version AS INT64) < 140, From c2a1c8ae7c4e3f8c9cafd099a03f344b05087ac3 Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Tue, 1 Jul 2025 19:45:18 -0400 Subject: [PATCH 03/19] Fix tests --- .../event_aggregates_suggest_v1/query.sql | 6 +++--- .../event_aggregates_v1/query.sql | 12 ++++++------ .../request_payload_suggest_v2/query.sql | 12 ++++++------ .../suggest_impression_sanitized_v3/query.sql | 12 ++++++------ .../event_aggregates_v1/test_aggregation/expect.yaml | 12 ++++++++---- ...ta-shared-prod.firefox_desktop.quick_suggest.yaml | 11 +++++++++++ ...ta-shared-prod.firefox_desktop.quick_suggest.yaml | 2 ++ 7 files changed, 42 insertions(+), 25 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql index 264ede83fb1..771f7b06d78 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql @@ -18,9 +18,9 @@ combined AS ( -- receives geo information from the client rather than from Glean ingestion's -- IP geolocation. We no longer send subdivision, only country. IF( - SAFE_CAST(metadata.user_agent.version AS INT64) < 140, - normalized_country_code, - metrics.string.quick_suggest_country + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + metrics.string.quick_suggest_country, + normalized_country_code ) AS country, LOWER(metrics.string.quick_suggest_advertiser) AS advertiser, SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql index d15eb3cbb41..e222df28c63 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql @@ -24,14 +24,14 @@ combined AS ( -- receives geo information from the client rather than from Glean ingestion's -- IP geolocation. We no longer send subdivision, only country. IF( - SAFE_CAST(metadata.user_agent.version AS INT64) < 140, - normalized_country_code, - metrics.string.quick_suggest_country + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + metrics.string.quick_suggest_country, + normalized_country_code ) AS country, IF( - SAFE_CAST(metadata.user_agent.version AS INT64) < 140, - metadata.geo.subdivision1, - CAST(NULL AS STRING) + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + CAST(NULL AS STRING), + metadata.geo.subdivision1 ) AS subdivision1, metrics.string.quick_suggest_advertiser AS advertiser, client_info.app_channel AS release_channel, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql index 69e2e2ad9c8..4f4e73db2ed 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql @@ -22,14 +22,14 @@ ping_data AS ( -- receives geo information from the client rather than from Glean ingestion's -- IP geolocation. We no longer send subdivision, only country. IF( - SAFE_CAST(metadata.user_agent.version AS INT64) < 140, - metadata.geo.country, - metrics.string.quick_suggest_country + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + metrics.string.quick_suggest_country, + metadata.geo.country ) AS country_code, IF( - SAFE_CAST(metadata.user_agent.version AS INT64) < 140, - metadata.geo.subdivision1, - CAST(NULL AS STRING) + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + CAST(NULL AS STRING), + metadata.geo.subdivision1 ) AS region_code, metadata.user_agent.os AS os_family, metadata.user_agent.version AS product_version, diff --git a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql index 9eaf2649118..c312a15ae76 100644 --- a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql +++ b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql @@ -46,14 +46,14 @@ WITH impressions AS ( metrics.boolean.quick_suggest_is_clicked AS is_clicked, client_info.locale AS locale, IF( - SAFE_CAST(metadata.user_agent.version AS INT64) < 140, - metadata.geo.country, - metrics.string.quick_suggest_country + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + metrics.string.quick_suggest_country, + metadata.geo.country ) AS country, IF( - SAFE_CAST(metadata.user_agent.version AS INT64) < 140, - metadata.geo.subdivision1, - CAST(NULL AS STRING) + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + CAST(NULL AS STRING), + metadata.geo.subdivision1 ) AS region, normalized_os, normalized_os_version, diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml index 0a0e3cda839..e19741ea91c 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml @@ -3,10 +3,9 @@ submission_date: "2030-01-01" source: suggest provider: merino - event_type: impression + event_type: click form_factor: desktop - country: US - subdivision1: AZ + country: DE advertiser: ad1 release_channel: release match_type: "firefox-suggest" @@ -17,7 +16,12 @@ user_count: 1 query_type: branded - <<: *suggest_base - event_type: click + event_type: impression + country: US + subdivision1: AZ +- <<: *suggest_base + country: US + subdivision1: AZ - &suggest_mobile submission_date: "2030-01-01" source: suggest diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml index a43b7ec3802..13b41b242fc 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml @@ -8,6 +8,7 @@ subdivision1: AZ user_agent: os: "Windows 11" + version: 130 metrics: &click_metrics boolean: quick_suggest_improve_suggest_experience: true @@ -28,3 +29,13 @@ string: <<: *click_strings quick_suggest_ping_type: quicksuggest-impression +- <<: *click + metadata: + user_agent: + os: "Windows 11" + version: 141 + metrics: + <<: *click_metrics + string: + <<: *click_strings + quick_suggest_country: DE diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml index 4ce20e7eed5..2e0cd766d8a 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml @@ -8,6 +8,7 @@ subdivision1: AZ user_agent: os: "Windows 11" + version: 130 metrics: &click_metrics boolean: quick_suggest_improve_suggest_experience: true @@ -19,6 +20,7 @@ quick_suggest_match_type: firefox-suggest quick_suggest_ping_type: quicksuggest-click quick_suggest_request_id: HASH123 + quick_suggest_country: null uuid: quick_suggest_context_id: a normalized_country_code: US From 8bb47a9a58fd111a0cc489017fc8994b3114d57b Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Wed, 9 Jul 2025 13:48:37 -0400 Subject: [PATCH 04/19] Update suggest_impression_sanitized_v3 test --- .../event_aggregates_suggest_v1/query.sql | 2 +- .../event_aggregates_v1/query.sql | 4 +- .../request_payload_suggest_v2/query.sql | 4 +- .../suggest_impression_sanitized_v3/query.sql | 4 +- .../test_single_day/expect.yaml | 21 +++++++++++ ...refox_desktop_stable.quick_suggest_v1.yaml | 37 +++++++++++++++++++ ...terms_derived.merino_log_sanitized_v3.yaml | 11 ++++++ 7 files changed, 76 insertions(+), 7 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql index 771f7b06d78..01948d5fe35 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql @@ -18,7 +18,7 @@ combined AS ( -- receives geo information from the client rather than from Glean ingestion's -- IP geolocation. We no longer send subdivision, only country. IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, metrics.string.quick_suggest_country, normalized_country_code ) AS country, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql index e222df28c63..32cc4c7e91b 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql @@ -24,12 +24,12 @@ combined AS ( -- receives geo information from the client rather than from Glean ingestion's -- IP geolocation. We no longer send subdivision, only country. IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, metrics.string.quick_suggest_country, normalized_country_code ) AS country, IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, CAST(NULL AS STRING), metadata.geo.subdivision1 ) AS subdivision1, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql index 4f4e73db2ed..313be3c0d3e 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql @@ -22,12 +22,12 @@ ping_data AS ( -- receives geo information from the client rather than from Glean ingestion's -- IP geolocation. We no longer send subdivision, only country. IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, metrics.string.quick_suggest_country, metadata.geo.country ) AS country_code, IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, CAST(NULL AS STRING), metadata.geo.subdivision1 ) AS region_code, diff --git a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql index c312a15ae76..1be2e2ea884 100644 --- a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql +++ b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql @@ -46,12 +46,12 @@ WITH impressions AS ( metrics.boolean.quick_suggest_is_clicked AS is_clicked, client_info.locale AS locale, IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, metrics.string.quick_suggest_country, metadata.geo.country ) AS country, IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, + SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, CAST(NULL AS STRING), metadata.geo.subdivision1 ) AS region, diff --git a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/expect.yaml b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/expect.yaml index c263906e6a2..bf44814f31a 100644 --- a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/expect.yaml +++ b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/expect.yaml @@ -116,3 +116,24 @@ reporting_url: https://example.com/duff-beer submission_timestamp: "2021-01-01T13:03:00+00:00" version: "134" +- advertiser: omni consumer products + block_id: 205 + browser: firefox(141) + context_id: cid15 + country: US + form_factor: desktop + is_clicked: false + locale: en-US + normalized_channel: release + normalized_os: windows + normalized_os_version: 15.2 + os_family: windows + position: 15 + query: robot police + reporting_url: https://example.com/ocp + request_id: rid15 + sequence_no: 15 + session_id: sid15 + submission_timestamp: "2021-01-01T13:05:00+00:00" + timestamp: "2021-01-01T13:05:00+00:00" + version: "141" diff --git a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml index 6038fd9b6a4..1a4fed7d3d1 100644 --- a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml +++ b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml @@ -6,6 +6,8 @@ geo: country: US subdivision1: CA + user_agent: + version: "134.0.1" metrics: boolean: quick_suggest_is_clicked: false @@ -32,6 +34,8 @@ geo: country: US subdivision1: PA + user_agent: + version: "134.0.1" metrics: boolean: quick_suggest_is_clicked: true @@ -58,6 +62,8 @@ geo: country: US subdivision1: OR + user_agent: + version: "134.0.1" metrics: boolean: quick_suggest_is_clicked: false @@ -84,6 +90,8 @@ geo: country: US subdivision1: WA + user_agent: + version: "134.0.1" metrics: boolean: quick_suggest_is_clicked: true @@ -102,3 +110,32 @@ normalized_os: Windows normalized_os_version: "11.0" sample_id: null +- submission_timestamp: "2021-01-01 13:05:00" + client_info: + app_display_version: "141.0.1" + locale: en-US + metadata: + geo: + country: null + subdivision1: null + user_agent: + version: "141.0.1" + metrics: + boolean: + quick_suggest_is_clicked: false + quantity: + quick_suggest_position: 15 + string: + quick_suggest_advertiser: omni consumer products + quick_suggest_block_id: 205 + quick_suggest_ping_type: "quicksuggest-impression" + quick_suggest_request_id: rid15 + quick_suggest_country: US + url2: + quick_suggest_reporting_url: "https://example.com/ocp" + uuid: + quick_suggest_context_id: cid15 + normalized_channel: release + normalized_os: windows + normalized_os_version: "15.2" + sample_id: null diff --git a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.search_terms_derived.merino_log_sanitized_v3.yaml b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.search_terms_derived.merino_log_sanitized_v3.yaml index 4cfc2e3a6e9..60b0c244421 100644 --- a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.search_terms_derived.merino_log_sanitized_v3.yaml +++ b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.search_terms_derived.merino_log_sanitized_v3.yaml @@ -64,3 +64,14 @@ form_factor: desktop browser: firefox(134) os_family: windows +- timestamp: "2021-01-01 13:05:00.000000" + request_id: rid15 + session_id: sid15 + sequence_no: 15 + query: "robot police" + country: null + region: null + dma: null + form_factor: desktop + browser: firefox(141) + os_family: windows From d17148848cd245f096182801dc0b50a9989a5489 Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Wed, 9 Jul 2025 14:18:46 -0400 Subject: [PATCH 05/19] update CODEOWNERS --- CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 4d2f7c6b135..68f8709c9ed 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -11,8 +11,8 @@ /sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/ @mozilla/revenue_forecasting_data_reviewers /sql/moz-fx-data-shared-prod/search_derived/search_dau_aggregates_v1/ @mozilla/revenue_forecasting_data_reviewers # Contextual Services -/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2 @mozilla/request_payload_reviewers -/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2 @mozilla/request_payload_reviewers +/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/ @mozilla/request_payload_reviewers +/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2/ @mozilla/request_payload_reviewers /sql/moz-fx-data-shared-prod/contextual_services_derived/suggest_revenue_levers_daily_v1 @mozilla/revenue_forecasting_data_reviewers /sql/moz-fx-data-shared-prod/monitoring_derived/jobs_by_organization_v1 @mozilla/dataops # Dependency updates From 01f5dc36841ab3ecd8b2deac329a2d8b42f7b263 Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Wed, 9 Jul 2025 14:53:16 -0400 Subject: [PATCH 06/19] Fix cast --- .../event_aggregates_suggest_v1/query.sql | 2 +- .../event_aggregates_v1/query.sql | 4 ++-- .../request_payload_suggest_v2/query.sql | 4 ++-- .../suggest_impression_sanitized_v3/query.sql | 4 ++-- ...data-shared-prod.firefox_desktop.quick_suggest.yaml | 4 ++-- ...d-prod.firefox_desktop_stable.quick_suggest_v1.yaml | 10 +++++----- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql index 01948d5fe35..771f7b06d78 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql @@ -18,7 +18,7 @@ combined AS ( -- receives geo information from the client rather than from Glean ingestion's -- IP geolocation. We no longer send subdivision, only country. IF( - SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, metrics.string.quick_suggest_country, normalized_country_code ) AS country, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql index 32cc4c7e91b..e222df28c63 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql @@ -24,12 +24,12 @@ combined AS ( -- receives geo information from the client rather than from Glean ingestion's -- IP geolocation. We no longer send subdivision, only country. IF( - SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, metrics.string.quick_suggest_country, normalized_country_code ) AS country, IF( - SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, CAST(NULL AS STRING), metadata.geo.subdivision1 ) AS subdivision1, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql index 313be3c0d3e..4f4e73db2ed 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql @@ -22,12 +22,12 @@ ping_data AS ( -- receives geo information from the client rather than from Glean ingestion's -- IP geolocation. We no longer send subdivision, only country. IF( - SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, metrics.string.quick_suggest_country, metadata.geo.country ) AS country_code, IF( - SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, CAST(NULL AS STRING), metadata.geo.subdivision1 ) AS region_code, diff --git a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql index 1be2e2ea884..c312a15ae76 100644 --- a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql +++ b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql @@ -46,12 +46,12 @@ WITH impressions AS ( metrics.boolean.quick_suggest_is_clicked AS is_clicked, client_info.locale AS locale, IF( - SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, metrics.string.quick_suggest_country, metadata.geo.country ) AS country, IF( - SAFE_CAST(SPLIT(metadata.user_agent.version, '.')[0] AS INT64) >= 140, + SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, CAST(NULL AS STRING), metadata.geo.subdivision1 ) AS region, diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml index 13b41b242fc..0d516afa7d2 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml @@ -8,7 +8,7 @@ subdivision1: AZ user_agent: os: "Windows 11" - version: 130 + version: "130" metrics: &click_metrics boolean: quick_suggest_improve_suggest_experience: true @@ -33,7 +33,7 @@ metadata: user_agent: os: "Windows 11" - version: 141 + version: "141" metrics: <<: *click_metrics string: diff --git a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml index 1a4fed7d3d1..6d6614826cc 100644 --- a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml +++ b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml @@ -7,7 +7,7 @@ country: US subdivision1: CA user_agent: - version: "134.0.1" + version: "134" metrics: boolean: quick_suggest_is_clicked: false @@ -35,7 +35,7 @@ country: US subdivision1: PA user_agent: - version: "134.0.1" + version: "134" metrics: boolean: quick_suggest_is_clicked: true @@ -63,7 +63,7 @@ country: US subdivision1: OR user_agent: - version: "134.0.1" + version: "134" metrics: boolean: quick_suggest_is_clicked: false @@ -91,7 +91,7 @@ country: US subdivision1: WA user_agent: - version: "134.0.1" + version: "134" metrics: boolean: quick_suggest_is_clicked: true @@ -119,7 +119,7 @@ country: null subdivision1: null user_agent: - version: "141.0.1" + version: "141" metrics: boolean: quick_suggest_is_clicked: false From 6d2d8ad64c5afc559c5589274c32b22c598ee20d Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Wed, 9 Jul 2025 15:41:17 -0400 Subject: [PATCH 07/19] Update codeowners --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 68f8709c9ed..bb7a867d762 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -11,7 +11,7 @@ /sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/ @mozilla/revenue_forecasting_data_reviewers /sql/moz-fx-data-shared-prod/search_derived/search_dau_aggregates_v1/ @mozilla/revenue_forecasting_data_reviewers # Contextual Services -/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/ @mozilla/request_payload_reviewers +/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/ akommasani@mozilla.com skahmann@mozilla.com @mozilla/request_payload_reviewers /sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2/ @mozilla/request_payload_reviewers /sql/moz-fx-data-shared-prod/contextual_services_derived/suggest_revenue_levers_daily_v1 @mozilla/revenue_forecasting_data_reviewers /sql/moz-fx-data-shared-prod/monitoring_derived/jobs_by_organization_v1 @mozilla/dataops From c883dba5b4930c9ef4d83d3ad6d33d5154aed274 Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Wed, 9 Jul 2025 16:12:58 -0400 Subject: [PATCH 08/19] Undo changes to CODEOWNERS --- CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index bb7a867d762..4d2f7c6b135 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -11,8 +11,8 @@ /sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/ @mozilla/revenue_forecasting_data_reviewers /sql/moz-fx-data-shared-prod/search_derived/search_dau_aggregates_v1/ @mozilla/revenue_forecasting_data_reviewers # Contextual Services -/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/ akommasani@mozilla.com skahmann@mozilla.com @mozilla/request_payload_reviewers -/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2/ @mozilla/request_payload_reviewers +/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2 @mozilla/request_payload_reviewers +/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2 @mozilla/request_payload_reviewers /sql/moz-fx-data-shared-prod/contextual_services_derived/suggest_revenue_levers_daily_v1 @mozilla/revenue_forecasting_data_reviewers /sql/moz-fx-data-shared-prod/monitoring_derived/jobs_by_organization_v1 @mozilla/dataops # Dependency updates From 2ac938183cfd6a58b42274daaba5ae7655831c03 Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Wed, 9 Jul 2025 16:16:43 -0400 Subject: [PATCH 09/19] Test CI codeowners check --- .../active_users_aggregates_v4/templates/mobile_query.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql_generators/active_users_aggregates_v4/templates/mobile_query.sql b/sql_generators/active_users_aggregates_v4/templates/mobile_query.sql index 7a8b6d91aed..c53077765eb 100644 --- a/sql_generators/active_users_aggregates_v4/templates/mobile_query.sql +++ b/sql_generators/active_users_aggregates_v4/templates/mobile_query.sql @@ -5,7 +5,7 @@ WITH SELECT client_id, adjust_network, - install_source + install_source, FROM fenix.firefox_android_clients ), From bccfc9497b3638a1d95fa667eb6c7dbb07397051 Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Fri, 11 Jul 2025 15:11:12 -0400 Subject: [PATCH 10/19] Undo change --- .../active_users_aggregates_v4/templates/mobile_query.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql_generators/active_users_aggregates_v4/templates/mobile_query.sql b/sql_generators/active_users_aggregates_v4/templates/mobile_query.sql index c53077765eb..7a8b6d91aed 100644 --- a/sql_generators/active_users_aggregates_v4/templates/mobile_query.sql +++ b/sql_generators/active_users_aggregates_v4/templates/mobile_query.sql @@ -5,7 +5,7 @@ WITH SELECT client_id, adjust_network, - install_source, + install_source FROM fenix.firefox_android_clients ), From 7d45317fc66ccbbd52edfba069760f9020c264be Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Thu, 21 Aug 2025 18:41:20 -0400 Subject: [PATCH 11/19] Simplify logic: just fall back to metrics.string.quick_suggest_country --- .../event_aggregates_suggest_v1/query.sql | 12 ++++++------ .../event_aggregates_v1/query.sql | 14 +++----------- .../request_payload_suggest_v2/query.sql | 14 +++----------- .../suggest_impression_sanitized_v3/query.sql | 15 +++++---------- ...shared-prod.firefox_desktop.quick_suggest.yaml | 5 +++-- 5 files changed, 20 insertions(+), 40 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql index 771f7b06d78..d68200a1a5e 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql @@ -14,14 +14,10 @@ combined AS ( metrics.uuid.quick_suggest_context_id AS context_id, DATE(submission_timestamp) AS submission_date, 'desktop' AS form_factor, - -- As of Firefox 140, the quick_suggest ping is sent via OHTTP and now + -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now -- receives geo information from the client rather than from Glean ingestion's -- IP geolocation. We no longer send subdivision, only country. - IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, - metrics.string.quick_suggest_country, - normalized_country_code - ) AS country, + COALESCE(normalized_country_code, metrics.string.quick_suggest_country) AS country, LOWER(metrics.string.quick_suggest_advertiser) AS advertiser, SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os, client_info.app_channel AS release_channel, @@ -55,6 +51,8 @@ combined AS ( DATE(submission_timestamp) AS submission_date, 'desktop' AS form_factor, normalized_country_code AS country, + normalized_country_code AS country_ip, + CAST(NULL AS STRING) AS country_browser, LOWER(advertiser) AS advertiser, SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os, release_channel, @@ -82,6 +80,8 @@ combined AS ( DATE(submission_timestamp) AS submission_date, 'desktop' AS form_factor, normalized_country_code AS country, + normalized_country_code AS country_ip, + CAST(NULL AS STRING) AS country_browser, LOWER(advertiser) AS advertiser, SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os, release_channel, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql index e222df28c63..77ea13aaed0 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql @@ -20,19 +20,11 @@ combined AS ( "impression" ) AS event_type, 'desktop' AS form_factor, - -- As of Firefox 140, the quick_suggest ping is sent via OHTTP and now + -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now -- receives geo information from the client rather than from Glean ingestion's -- IP geolocation. We no longer send subdivision, only country. - IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, - metrics.string.quick_suggest_country, - normalized_country_code - ) AS country, - IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, - CAST(NULL AS STRING), - metadata.geo.subdivision1 - ) AS subdivision1, + COALESCE(normalized_country_code, metrics.string.quick_suggest_country) AS country, + metadata.geo.subdivision1 AS subdivision1, metrics.string.quick_suggest_advertiser AS advertiser, client_info.app_channel AS release_channel, metrics.quantity.quick_suggest_position AS position, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql index 4f4e73db2ed..eeae79f2e21 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql @@ -18,19 +18,11 @@ ping_data AS ( "click", "impression" ) AS interaction_type, - -- As of Firefox 140, the quick_suggest ping is sent via OHTTP and now + -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now -- receives geo information from the client rather than from Glean ingestion's -- IP geolocation. We no longer send subdivision, only country. - IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, - metrics.string.quick_suggest_country, - metadata.geo.country - ) AS country_code, - IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, - CAST(NULL AS STRING), - metadata.geo.subdivision1 - ) AS region_code, + COALESCE(metadata.geo.country, metrics.string.quick_suggest_country) AS country, + metadata.geo.subdivision1 AS region_code, metadata.user_agent.os AS os_family, metadata.user_agent.version AS product_version, FROM diff --git a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql index c312a15ae76..4b87d9bc994 100644 --- a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql +++ b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql @@ -45,16 +45,11 @@ WITH impressions AS ( sample_id, metrics.boolean.quick_suggest_is_clicked AS is_clicked, client_info.locale AS locale, - IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, - metrics.string.quick_suggest_country, - metadata.geo.country - ) AS country, - IF( - SAFE_CAST(metadata.user_agent.version AS INT64) >= 140, - CAST(NULL AS STRING), - metadata.geo.subdivision1 - ) AS region, + -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now + -- receives geo information from the client rather than from Glean ingestion's + -- IP geolocation. We no longer send subdivision, only country. + COALESCE(metadata.geo.country, metrics.string.quick_suggest_country) AS country, + metadata.geo.subdivision1 AS region, normalized_os, normalized_os_version, normalized_channel, diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml index 0d516afa7d2..2469bca6e5d 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.firefox_desktop.quick_suggest.yaml @@ -8,7 +8,6 @@ subdivision1: AZ user_agent: os: "Windows 11" - version: "130" metrics: &click_metrics boolean: quick_suggest_improve_suggest_experience: true @@ -31,11 +30,13 @@ quick_suggest_ping_type: quicksuggest-impression - <<: *click metadata: + geo: + subdivision1: null user_agent: os: "Windows 11" - version: "141" metrics: <<: *click_metrics string: <<: *click_strings quick_suggest_country: DE + normalized_country_code: null From 074014e650fdcc351dd5f29b5dbc11a6597d6a56 Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Thu, 21 Aug 2025 18:42:48 -0400 Subject: [PATCH 12/19] Remove country_ip and country_browser fields --- .../event_aggregates_suggest_v1/query.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql index d68200a1a5e..2869face2d7 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql @@ -51,8 +51,6 @@ combined AS ( DATE(submission_timestamp) AS submission_date, 'desktop' AS form_factor, normalized_country_code AS country, - normalized_country_code AS country_ip, - CAST(NULL AS STRING) AS country_browser, LOWER(advertiser) AS advertiser, SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os, release_channel, @@ -80,8 +78,6 @@ combined AS ( DATE(submission_timestamp) AS submission_date, 'desktop' AS form_factor, normalized_country_code AS country, - normalized_country_code AS country_ip, - CAST(NULL AS STRING) AS country_browser, LOWER(advertiser) AS advertiser, SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os, release_channel, From 99c7bc9fb4208345edc78872c9bb7747bd246948 Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Thu, 21 Aug 2025 18:45:10 -0400 Subject: [PATCH 13/19] Remove unneeded version --- ...d-prod.firefox_desktop_stable.quick_suggest_v1.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml index 6d6614826cc..705f394ab47 100644 --- a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml +++ b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml @@ -6,8 +6,6 @@ geo: country: US subdivision1: CA - user_agent: - version: "134" metrics: boolean: quick_suggest_is_clicked: false @@ -34,8 +32,6 @@ geo: country: US subdivision1: PA - user_agent: - version: "134" metrics: boolean: quick_suggest_is_clicked: true @@ -62,8 +58,6 @@ geo: country: US subdivision1: OR - user_agent: - version: "134" metrics: boolean: quick_suggest_is_clicked: false @@ -90,8 +84,6 @@ geo: country: US subdivision1: WA - user_agent: - version: "134" metrics: boolean: quick_suggest_is_clicked: true @@ -118,8 +110,6 @@ geo: country: null subdivision1: null - user_agent: - version: "141" metrics: boolean: quick_suggest_is_clicked: false From cdd7c75b096d98641f0a8c3c868aaa642a936dae Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Thu, 21 Aug 2025 20:10:19 -0400 Subject: [PATCH 14/19] Fix tests --- .../event_aggregates_suggest_v1/query.sql | 3 ++- .../event_aggregates_v1/query.sql | 3 ++- .../test_aggregation/expect.yaml | 14 ++++++++++++++ .../moz-fx-data-shared-prod.fenix.fx_suggest.yaml | 14 ++++++++++++++ .../moz-fx-data-shared-prod.fenix.fx_suggest.yaml | 1 + 5 files changed, 33 insertions(+), 2 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql index 2869face2d7..6b686b1f03f 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/query.sql @@ -105,7 +105,8 @@ combined AS ( metrics.uuid.fx_suggest_context_id AS context_id, DATE(submission_timestamp) AS submission_date, 'phone' AS form_factor, - normalized_country_code AS country, + -- With shift to OHTTP, we expect to stop receiving normalized_country_code soon + COALESCE(normalized_country_code, metrics.string.fx_suggest_country) AS country, metrics.string.fx_suggest_advertiser AS advertiser, SPLIT(metadata.user_agent.os, ' ')[SAFE_OFFSET(0)] AS normalized_os, client_info.app_channel AS release_channel, diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql index 77ea13aaed0..065e6a5c53c 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/query.sql @@ -120,7 +120,8 @@ combined AS ( "impression" ) AS event_type, 'phone' AS form_factor, - normalized_country_code AS country, + -- With shift to OHTTP, we expect to stop receiving normalized_country_code soon + COALESCE(normalized_country_code, metrics.string.fx_suggest_country) AS country, metadata.geo.subdivision1 AS subdivision1, metrics.string.fx_suggest_advertiser AS advertiser, client_info.app_channel AS release_channel, diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml index e19741ea91c..d8b41b82473 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/expect.yaml @@ -40,6 +40,20 @@ query_type: branded - <<: *suggest_mobile normalized_os: iOS +- submission_date: "2030-01-01" + source: suggest + provider: remote settings + event_type: impression + form_factor: phone + country: VN + advertiser: ad4 + release_channel: release + match_type: "firefox-suggest" + normalized_os: Android + position: 1 + event_count: 1 + user_count: 1 + query_type: branded - &topsites_base submission_date: "2030-01-01" source: topsites diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.fenix.fx_suggest.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.fenix.fx_suggest.yaml index 96f04b1f531..58ffa2630d4 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.fenix.fx_suggest.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_aggregation/moz-fx-data-shared-prod.fenix.fx_suggest.yaml @@ -8,6 +8,7 @@ string: fx_suggest_advertiser: ad4 fx_suggest_ping_type: fxsuggest-impression + fx_suggest_country: null uuid: fx_suggest_context_id: fenix-b metadata: @@ -18,6 +19,19 @@ client_info: app_channel: release normalized_country_code: US +- <<: *suggest_base + metrics: + <<: *suggest_metrics + string: + fx_suggest_advertiser: ad4 + fx_suggest_ping_type: fxsuggest-impression + fx_suggest_country: VN + metadata: + geo: + subdivision1: null + user_agent: + os: Android + normalized_country_code: null - <<: *suggest_base metrics: <<: *suggest_metrics diff --git a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.fenix.fx_suggest.yaml b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.fenix.fx_suggest.yaml index 1e68a463af1..28978c36cde 100644 --- a/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.fenix.fx_suggest.yaml +++ b/tests/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/test_overactive_filter/moz-fx-data-shared-prod.fenix.fx_suggest.yaml @@ -8,6 +8,7 @@ string: fx_suggest_advertiser: ad4 fx_suggest_ping_type: fxsuggest-click + fx_suggest_country: null uuid: fx_suggest_context_id: fenix-b metadata: From 6ba56880cd77643341067b7552e74f5b6596b4fa Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Wed, 27 Aug 2025 16:24:17 -0400 Subject: [PATCH 15/19] Fix CODEOWNERS --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index b9e270da411..7d830d39957 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -22,7 +22,7 @@ dags.yaml /sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/ @mozilla/revenue_forecasting_data_reviewers /sql/moz-fx-data-shared-prod/search_derived/search_dau_aggregates_v1/ @mozilla/revenue_forecasting_data_reviewers # Contextual Services -/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2 @mozilla/request_payload_reviewers +/sql/**/request_payload_suggest_v2 @mozilla/request_payload_reviewers /sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2 @mozilla/request_payload_reviewers /sql/moz-fx-data-shared-prod/contextual_services_derived/suggest_revenue_levers_daily_v1 @mozilla/revenue_forecasting_data_reviewers /sql/moz-fx-data-shared-prod/monitoring_derived/jobs_by_organization_v1 @mozilla/dataops From 72f84b731b40ecb4fa798776ffb9acdeeeea7a0c Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Wed, 27 Aug 2025 17:43:22 -0400 Subject: [PATCH 16/19] Remove change to request_payload_suggest_v2 --- .../request_payload_suggest_v2/query.sql | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql index eeae79f2e21..20dd0eb16e2 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql @@ -18,10 +18,7 @@ ping_data AS ( "click", "impression" ) AS interaction_type, - -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now - -- receives geo information from the client rather than from Glean ingestion's - -- IP geolocation. We no longer send subdivision, only country. - COALESCE(metadata.geo.country, metrics.string.quick_suggest_country) AS country, + metadata.geo.country AS country_code, metadata.geo.subdivision1 AS region_code, metadata.user_agent.os AS os_family, metadata.user_agent.version AS product_version, From a84ab1e46061ae1976e69c8d812158e89392808e Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Thu, 28 Aug 2025 11:22:56 -0400 Subject: [PATCH 17/19] Revert "Remove change to request_payload_suggest_v2" This reverts commit 72f84b731b40ecb4fa798776ffb9acdeeeea7a0c. --- .../request_payload_suggest_v2/query.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql index 20dd0eb16e2..eeae79f2e21 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/query.sql @@ -18,7 +18,10 @@ ping_data AS ( "click", "impression" ) AS interaction_type, - metadata.geo.country AS country_code, + -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now + -- receives geo information from the client rather than from Glean ingestion's + -- IP geolocation. We no longer send subdivision, only country. + COALESCE(metadata.geo.country, metrics.string.quick_suggest_country) AS country, metadata.geo.subdivision1 AS region_code, metadata.user_agent.os AS os_family, metadata.user_agent.version AS product_version, From 0e449f54e6990d643fa83e597dd670d8140b0bab Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Fri, 29 Aug 2025 17:28:08 -0400 Subject: [PATCH 18/19] Remove change to suggest_impression_sanitized_v3 --- .../suggest_impression_sanitized_v3/query.sql | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql index 4b87d9bc994..87a0759717d 100644 --- a/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql +++ b/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/query.sql @@ -45,10 +45,7 @@ WITH impressions AS ( sample_id, metrics.boolean.quick_suggest_is_clicked AS is_clicked, client_info.locale AS locale, - -- As of Firefox 141, the quick_suggest ping is sent via OHTTP and now - -- receives geo information from the client rather than from Glean ingestion's - -- IP geolocation. We no longer send subdivision, only country. - COALESCE(metadata.geo.country, metrics.string.quick_suggest_country) AS country, + metadata.geo.country, metadata.geo.subdivision1 AS region, normalized_os, normalized_os_version, From 06e4f2bcce65e34f6978b037ad4b688befee46da Mon Sep 17 00:00:00 2001 From: Curtis Morales Date: Fri, 29 Aug 2025 17:58:18 -0400 Subject: [PATCH 19/19] Remove changes to tests --- .../test_single_day/expect.yaml | 21 --------------- ...refox_desktop_stable.quick_suggest_v1.yaml | 27 ------------------- ...terms_derived.merino_log_sanitized_v3.yaml | 11 -------- 3 files changed, 59 deletions(-) diff --git a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/expect.yaml b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/expect.yaml index bf44814f31a..c263906e6a2 100644 --- a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/expect.yaml +++ b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/expect.yaml @@ -116,24 +116,3 @@ reporting_url: https://example.com/duff-beer submission_timestamp: "2021-01-01T13:03:00+00:00" version: "134" -- advertiser: omni consumer products - block_id: 205 - browser: firefox(141) - context_id: cid15 - country: US - form_factor: desktop - is_clicked: false - locale: en-US - normalized_channel: release - normalized_os: windows - normalized_os_version: 15.2 - os_family: windows - position: 15 - query: robot police - reporting_url: https://example.com/ocp - request_id: rid15 - sequence_no: 15 - session_id: sid15 - submission_timestamp: "2021-01-01T13:05:00+00:00" - timestamp: "2021-01-01T13:05:00+00:00" - version: "141" diff --git a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml index 705f394ab47..6038fd9b6a4 100644 --- a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml +++ b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.firefox_desktop_stable.quick_suggest_v1.yaml @@ -102,30 +102,3 @@ normalized_os: Windows normalized_os_version: "11.0" sample_id: null -- submission_timestamp: "2021-01-01 13:05:00" - client_info: - app_display_version: "141.0.1" - locale: en-US - metadata: - geo: - country: null - subdivision1: null - metrics: - boolean: - quick_suggest_is_clicked: false - quantity: - quick_suggest_position: 15 - string: - quick_suggest_advertiser: omni consumer products - quick_suggest_block_id: 205 - quick_suggest_ping_type: "quicksuggest-impression" - quick_suggest_request_id: rid15 - quick_suggest_country: US - url2: - quick_suggest_reporting_url: "https://example.com/ocp" - uuid: - quick_suggest_context_id: cid15 - normalized_channel: release - normalized_os: windows - normalized_os_version: "15.2" - sample_id: null diff --git a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.search_terms_derived.merino_log_sanitized_v3.yaml b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.search_terms_derived.merino_log_sanitized_v3.yaml index 60b0c244421..4cfc2e3a6e9 100644 --- a/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.search_terms_derived.merino_log_sanitized_v3.yaml +++ b/tests/sql/moz-fx-data-shared-prod/search_terms_derived/suggest_impression_sanitized_v3/test_single_day/moz-fx-data-shared-prod.search_terms_derived.merino_log_sanitized_v3.yaml @@ -64,14 +64,3 @@ form_factor: desktop browser: firefox(134) os_family: windows -- timestamp: "2021-01-01 13:05:00.000000" - request_id: rid15 - session_id: sid15 - sequence_no: 15 - query: "robot police" - country: null - region: null - dma: null - form_factor: desktop - browser: firefox(141) - os_family: windows