feat(DENG-11237): export referral totals CSV to GCS

bigquery_etl/referral_export/__init__.py

@@ -0,0 +1,98 @@
+"""Export referral install totals from BigQuery to a CSV file in GCS.
+
+Supports the Firefox Referral Program (DENG-11237). The Website team syncs the
+CSV into the Postgres DB behind the referral hub page on firefox.com.
+
+Output format matches the Website team's request: `invite_code,total_installs`
+(no header by default), one file per run date named `referral_data-<date>.csv`.
+The run date (not wall-clock time) is used so an Airflow retry overwrites the
+same object rather than accumulating a second file for the same logical date.
+"""
+
+import logging
+
+import rich_click as click
+from google.cloud import bigquery
+
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger(__name__)
+
+
+@click.command()
+@click.option(
+    "--source-project",
+    required=True,
+    help="Google Cloud Project where the source table is located.",
+)
+@click.option(
+    "--source-dataset",
+    required=True,
+    help="Dataset in BigQuery where the source table is located.",
+)
+@click.option(
+    "--source-table",
+    required=True,
+    help="Name of the source table in BigQuery.",
+)
+@click.option(
+    "--destination-bucket",
+    required=True,
+    help="Destination Google Cloud Storage bucket (name only, no gs:// prefix).",
+)
+@click.option(
+    "--destination-prefix",
+    required=False,
+    default="",
+    help="Optional prefix (subfolder) within the bucket. Omit to write to the "
+    "bucket root.",
+)
+@click.option(
+    "--date",
+    required=True,
+    help="Run date (YYYY-MM-DD), used in the output filename. Pass Airflow's "
+    "{{ds}} so retries overwrite the same object instead of accumulating files.",
+)
+@click.option(
+    "--include-header/--no-include-header",
+    default=False,
+    help="Whether to write a CSV header row. Defaults to no header per the "
+    "Website team's requested format.",
+)
+def export_referral_totals_to_gcs(
+    source_project: str,
+    source_dataset: str,
+    source_table: str,
+    destination_bucket: str,
+    destination_prefix: str,
+    date: str,
+    include_header: bool,
+):
+    """Extract the referral totals table to a CSV file in GCS, named by run date."""
+    client = bigquery.Client(source_project)
+
+    filename = f"referral_data-{date}.csv"
+    object_path = (
+        f"{destination_prefix.rstrip('/')}/{filename}"
+        if destination_prefix
+        else filename
+    )
+    destination_uri = f"gs://{destination_bucket}/{object_path}"
+
+    job_config = bigquery.job.ExtractJobConfig(
+        destination_format=bigquery.job.DestinationFormat.CSV,
+        print_header=include_header,
+    )
+
+    extract_job = client.extract_table(
+        source=f"{source_project}.{source_dataset}.{source_table}",
+        destination_uris=[destination_uri],
+        job_config=job_config,
+    )
+    try:
+        extract_job.result()  # Waits for the job to complete.
+    except Exception as e:
+        raise click.ClickException(
+            f"Export to {destination_uri} failed: {extract_job.errors}"
+        ) from e
+
+    log.info(f"Export successful: {destination_uri}")

dags.yaml

@@ -1917,9 +1917,10 @@ bqetl_firefox_referral:
     start_date: '2026-07-23'
   description: |
     Aggregates Firefox first_run installs by referral (invite) code for the
-    Firefox Referral Program (DENG-11237). Chain: referral_installs_daily_v1 ->
-    referral_installs_totals_v1. The CSV-to-GCS export is a fast-follow, pending
-    the Website team's bucket + IAM.
+    Firefox Referral Program (DENG-11237), then exports the cumulative totals to
+    the Website team's GCS bucket (fx-referral-data-prod) as CSV. Chain:
+    referral_installs_daily_v1 -> referral_installs_totals_v1 ->
+    referral_installs_totals_to_gcs_v1.
   repo: bigquery-etl
   schedule_interval: 0 4 * * *
   tags:

sql/moz-fx-data-shared-prod/firefox_referral_derived/referral_installs_totals_to_gcs_v1/metadata.yaml

@@ -0,0 +1,38 @@
+friendly_name: Referral Installs Totals to GCS
+description: |-
+  Extracts referral_installs_totals_v1 to a CSV file in the Website team's GCS
+  bucket for the Firefox Referral Program (DENG-11237). firefox.com syncs the
+  CSV into the Postgres DB behind the referral hub page.
+
+  CSV column contract (no header row), written to the bucket root:
+    invite_code   STRING   referral (invite) code, fxrefer: prefix stripped
+    total_installs INTEGER cumulative all-time distinct first_run installs
+  One file per run date named `referral_data-<date>.csv` (date = Airflow {{ds}}),
+  so an Airflow retry overwrites the same object. Runs daily.
+
+  This task only extracts to GCS; it does not logically build a queryable table.
+  A schema.yaml is shipped alongside because the --isolated stage-deploy CI
+  hard-fails to resolve a query.py artifact without one; a side effect is an
+  empty companion table (see schema.yaml for why destination_table: null can't
+  be used here).
+
+  Destination bucket `fx-referral-data-prod` (project moz-fx-springfield-prod) is
+  owned by the Website team; the Airflow workload service account
+  (default-workloads@moz-fx-data-airflow-gke-prod.iam.gserviceaccount.com) was
+  granted write access via mozilla/webservices-infra#11912.
+owners:
+- phlee@mozilla.com
+labels:
+  incremental: false
+  owner1: phlee
+scheduling:
+  dag_name: bqetl_firefox_referral
+  arguments:
+  - --source-project=moz-fx-data-shared-prod
+  - --source-dataset=firefox_referral_derived
+  - --source-table=referral_installs_totals_v1
+  - --destination-bucket=fx-referral-data-prod
+  - --date={{ds}}
+  referenced_tables:
+  - ['moz-fx-data-shared-prod', 'firefox_referral_derived', 'referral_installs_totals_v1']
+bigquery: null

sql/moz-fx-data-shared-prod/firefox_referral_derived/referral_installs_totals_to_gcs_v1/query.py

@@ -0,0 +1,6 @@
+"""Export referral install totals to a CSV file in the Website team's GCS bucket."""
+
+from bigquery_etl.referral_export import export_referral_totals_to_gcs
+
+if __name__ == "__main__":
+    export_referral_totals_to_gcs()

sql/moz-fx-data-shared-prod/firefox_referral_derived/referral_installs_totals_to_gcs_v1/schema.yaml

@@ -0,0 +1,27 @@
+# This task extracts referral_installs_totals_v1 to a CSV in GCS; it does not
+# logically build a queryable table. This schema.yaml is required so the
+# --isolated stage-deploy CI can resolve the query.py artifact: without it, the
+# resolver ladder hard-fails (a .py can't be dry-run, and SELECT */get_table on
+# the not-yet-existent table are denied), which fails the deploy rather than
+# skipping it.
+#
+# Tradeoff: because a schema resolves, deploy also creates an (empty, unused)
+# companion table `firefox_referral_derived.referral_installs_totals_to_gcs_v1`.
+# The clean way to suppress that is scheduling.destination_table: null, but that
+# is incompatible with a query.py task — Task.of_python_script sets
+# query_file_path after construction, so the __attrs_post_init__ check
+# ("One of destination_table or query_file_path must be specified") fails DAG
+# generation. Columns mirror referral_installs_totals_v1 and document the CSV.
+fields:
+- name: invite_code
+  type: STRING
+  mode: NULLABLE
+  description: >-
+    Referral (invite) code, `fxrefer:` prefix stripped (expected 17 chars).
+    First column of the exported CSV.
+- name: total_installs
+  type: INTEGER
+  mode: NULLABLE
+  description: >-
+    Cumulative all-time distinct Firefox first_run installs for this invite
+    code. Second column of the exported CSV.