Fix lifecycle contributor state accounting and add extra QA for sqlmesh metrics pipeline

warehouse/metrics_tools/factory/factory.py

@@ -314,7 +314,7 @@ def queue_query(name: str):
                     parent_query = queries[table_name]["rendered_query"]
                     raise MetricsCycle(
                         textwrap.dedent(
-                            f"""Cycle from {name} to {table_name}: 
+                            f"""Cycle from {name} to {table_name}:
                          ---
                         {name}:
                         {rendered_query.sql(dialect="duckdb", pretty=True)}
@@ -744,7 +744,7 @@ def generate_time_aggregation_model_for_rendered_query(
             or "funding" in query_config["table_name"]
             or "releases" in query_config["table_name"]
             or "worldchain_users_aggregation" in query_config["table_name"]
-            or "new_contributors" in query_config["table_name"]
+            or "first_time_contributors" in query_config["table_name"]
         ):
             ignored_rules.append("incrementalmustdefinenogapsaudit")
 

warehouse/oso_sqlmesh/models/metrics_factories.py

@@ -53,9 +53,6 @@ def no_gaps_audit_factory(config: MetricQueryConfig) -> MetricMetadataModifiers:
     if "worldchain_users_aggregation" in config["table_name"]:
         return {}
 
-    if "new_contributors" in config["table_name"]:
-        return {}
-
     if "data_category=blockchain" in config["additional_tags"]:
         options["ignore_before"] = constants.superchain_audit_start
         options["ignore_after"] = constants.superchain_audit_end
@@ -256,8 +253,8 @@ def add_project_and_collection_entity_category_tags(
             ),
             additional_tags=["data_category=code"],
         ),
-        "new_contributors": MetricQueryDef(
-            ref="code/new_contributors.sql",
+        "first_time_contributor": MetricQueryDef(
+            ref="code/first_time_contributor.sql",
             time_aggregations=[
                 "daily",
                 "weekly",
@@ -266,11 +263,10 @@ def add_project_and_collection_entity_category_tags(
                 # "biannually",
                 "yearly",
             ],
-            entity_types=["artifact", "project", "collection"],
             over_all_time=True,
             metadata=MetricMetadata(
-                display_name="New Contributors",
-                description="Metrics related to new GitHub contributors identified by author_association = 'FIRST_TIME_CONTRIBUTOR'",
+                display_name="First Time Contributors",
+                description="Metrics related to first time GitHub contributors",
             ),
             additional_tags=["data_category=code"],
         ),

warehouse/oso_sqlmesh/oso_metrics/code/first_time_contributor.sql

@@ -0,0 +1,28 @@
+with first_contribution as (
+    select
+        from_artifact_id,
+        min(bucket_day) as first_contribution_date
+    from oso.int_events_daily__github
+    where
+        event_type
+        in ('COMMIT_CODE', 'ISSUE_OPENED', 'PULL_REQUEST_OPENED', 'PULL_REQUEST_MERGED')
+    group by 1
+)
+select
+    @metrics_sample_date(events.bucket_day) as metrics_sample_date,
+    events.event_source,
+    events.to_artifact_id as to_artifact_id,
+    '' as from_artifact_id,
+    @metric_name() as metric,
+    count(distinct events.from_artifact_id) as amount
+from oso.int_events_daily__github as events
+inner join
+    first_contribution
+    on
+        events.from_artifact_id = first_contribution.from_artifact_id
+        and events.bucket_day = first_contribution.first_contribution_date
+where
+    events.event_type
+    in ('COMMIT_CODE', 'ISSUE_OPENED', 'PULL_REQUEST_OPENED', 'PULL_REQUEST_MERGED')
+    and events.bucket_day between @metrics_start('DATE') and @metrics_end('DATE')
+group by 1, metric, from_artifact_id, to_artifact_id, event_source

warehouse/oso_sqlmesh/oso_metrics/code/lifecycle.sql

@@ -32,11 +32,11 @@ with history as (
     COALESCE(
       MAX(
         CASE
-          WHEN classification.metric LIKE 'new_%' THEN amount
+          WHEN classification.metric LIKE 'first_time_%' THEN amount
         END
       ),
       0
-    ) as new,
+    ) as first_time,
     COALESCE(
       MAX(
         CASE
@@ -57,7 +57,7 @@ with history as (
     classification.event_source
 ),
 lifecycle as (
-  -- Churn is prev.active - (latest.active - latest.new - latest.resurrected)
+  -- Churn is prev.active - (latest.active - latest.first_time - latest.resurrected)
   select history.metrics_sample_date,
     history.event_source,
     @metrics_entity_type_col(
@@ -70,7 +70,7 @@ lifecycle as (
         table_alias := history
       ), event_source
       ORDER BY history.metrics_sample_date
-    ), 0) - (history.active - history.new - history.resurrected) as churn,
+    ), 0) - (history.active - history.first_time - history.resurrected) as churn,
     history.full - COALESCE(LAG(history.full) OVER (
       PARTITION BY @metrics_entity_type_col(
         'to_{entity_type}_id',
@@ -85,13 +85,13 @@ lifecycle as (
       ), event_source
       ORDER BY history.metrics_sample_date
     ), 0) as change_in_part_time_contributors,
-    history.new - COALESCE(LAG(history.new) OVER (
+    history.first_time - COALESCE(LAG(history.first_time) OVER (
       PARTITION BY @metrics_entity_type_col(
         'to_{entity_type}_id',
         table_alias := history
       ), event_source
       ORDER BY history.metrics_sample_date
-    ), 0) as change_in_new_contributors,
+    ), 0) as change_in_first_time_contributors,
     history.active - COALESCE(LAG(history.active) OVER (
       PARTITION BY @metrics_entity_type_col(
         'to_{entity_type}_id',
@@ -111,8 +111,8 @@ select lifecycle.metrics_sample_date,
   ),
   lifecycle.event_source,
   '' as from_artifact_id,
-  @metrics_name('change_in_new_contributors') as metric,
-  lifecycle.change_in_new_contributors as amount
+  @metrics_name('change_in_first_time_contributors') as metric,
+  lifecycle.change_in_first_time_contributors as amount
 from lifecycle as lifecycle
 union all
 select lifecycle.metrics_sample_date,