Add measures to metrics (temporary) & fix metric deduplication

dbt_semantic_interfaces/implementations/metric.py

@@ -350,6 +350,7 @@ def build_metric_aggregation_params(
         return PydanticMetricAggregationParams(
             semantic_model=semantic_model_name,
             agg=measure.agg,
+            # TODO: clone this
             agg_params=measure.agg_params,
             agg_time_dimension=measure.agg_time_dimension,
             non_additive_dimension=measure.non_additive_dimension,

dbt_semantic_interfaces/transformations/add_input_metric_measures.py

@@ -31,9 +31,7 @@ def _get_measures_for_metric(
     ) -> Set[PydanticMetricInputMeasure]:
         """Returns a unique set of input measures for a given metric."""
         measures: Set = set()
-        matched_metric = next(
-            iter((metric for metric in semantic_manifest.metrics if metric.name == metric_name)), None
-        )
+        matched_metric = next((metric for metric in semantic_manifest.metrics if metric.name == metric_name), None)
         if matched_metric:
             if matched_metric.type is MetricType.SIMPLE or matched_metric.type is MetricType.CUMULATIVE:
                 if matched_metric.type_params.measure is not None:

dbt_semantic_interfaces/transformations/measure_to_metric_transformation_pieces/measure_features_to_metric_name.py

@@ -3,6 +3,7 @@
 from dbt_semantic_interfaces.implementations.elements.measure import PydanticMeasure
 from dbt_semantic_interfaces.implementations.metric import (
     PydanticMetric,
+    PydanticMetricInputMeasure,
     PydanticMetricTypeParams,
 )
 from dbt_semantic_interfaces.implementations.semantic_manifest import (
@@ -65,16 +66,31 @@ def _find_metric_clone_in_manifest(
         but let's not prematurely optimize.
         """
         search_metric = metric.copy(deep=True)
+        original_input_measures = search_metric.type_params.input_measures
+        original_measure = search_metric.type_params.measure
         for existing_metric in manifest.metrics:
             # this allows us to a straight equality comparison, which is safer in the future
             # than implementing a custom comparison function.
             search_metric.name = existing_metric.name
+            search_metric.description = existing_metric.description
+            search_metric.label = existing_metric.label
             search_metric.metadata = existing_metric.metadata
             search_metric.type_params.is_private = existing_metric.type_params.is_private
+            search_metric.config = existing_metric.config
+            # this is used for compatibility between new and old style metrics; we need to
+            # match if the search input measures are empty (new-style) or if they match
+            # the old-style input measures.
+            if len(existing_metric.type_params.input_measures) == 0:
+                search_metric.type_params.input_measures = []
+            else:
+                search_metric.type_params.input_measures = original_input_measures
+            if existing_metric.type_params.measure is None:
+                search_metric.type_params.measure = None
+            else:
+                search_metric.type_params.measure = original_measure
+
             if search_metric == existing_metric:
                 return existing_metric
-            print("provided metric", search_metric)
-            print("existing metric", existing_metric)
         return None
 
     @staticmethod
@@ -83,7 +99,7 @@ def build_metric_from_measure_configuration(
         semantic_model_name: str,
         fill_nulls_with: Optional[int],
         join_to_timespine: Optional[bool],
-        is_private: bool = True,
+        is_private: bool,
     ) -> PydanticMetric:
         """Build a metric from the measure configuration.
 
@@ -176,7 +192,12 @@ def get_or_create_metric_for_measure(
             semantic_model_name=model_name,
             fill_nulls_with=fill_nulls_with,
             join_to_timespine=join_to_timespine,
+            is_private=True,
         )
+        # supporting legacy cases.  Remove when we can remove input measures.
+        built_metric.type_params.measure = PydanticMetricInputMeasure(name=measure.name)
+        built_metric.type_params.input_measures = [PydanticMetricInputMeasure(name=measure.name)]
+
         metric = self._find_metric_clone_in_manifest(
             metric=built_metric,
             manifest=manifest,
@@ -193,6 +214,9 @@ def get_or_create_metric_for_measure(
             )
             metric = built_metric
             metric.name = metric_name
+            # TODO: remove this line when MF can support no-measure metrics
+            metric.type_params.measure = PydanticMetricInputMeasure(name=measure.name)
+            metric.type_params.input_measures.append(metric.type_params.measure)
             manifest.metrics.append(metric)
             existing_metric_names.add(metric_name)
 

tests/parsing/test_metric_parsing_with_custom_grain.py

@@ -48,6 +48,7 @@ def test_cumulative_metric_with_custom_grain_to_date() -> None:  # noqa: D
           type_params:
             measure:
               name: bookings
+              fill_nulls_with: 15
             cumulative_type_params:
               grain_to_date: martian_week
         """
@@ -121,9 +122,9 @@ def test_cumulative_metric_with_custom_window() -> None:  # noqa: D
     )
     assert not model.issues.has_blocking_issues
     semantic_manifest = model.semantic_manifest
-    # 2 explicit ones and one that is created for the measure input for the
-    # cumulative metric's params
-    assert len(semantic_manifest.metrics) == 3
+    # 2 explicit metrics.  The cumulative metric's input metric should be deduplicated
+    # so it will match.
+    assert len(semantic_manifest.metrics) == 2
 
     metric = next((m for m in semantic_manifest.metrics if m.name == "test_cumulative_metric_with_custom_window"), None)
     assert metric is not None, "Can't find metric"
@@ -185,9 +186,9 @@ def test_conversion_metric_with_custom_grain_window() -> None:  # noqa: D
     )
     assert not model.issues.has_blocking_issues
     semantic_manifest = model.semantic_manifest
-    # 2 explicit ones and one that is created for the measure input for the
-    # cumulative metric's params
-    assert len(semantic_manifest.metrics) == 3
+    # 2 explicitly created metrics.  The conversion measure -> metric transformation
+    # should not need to create a new metric since the existing one already matches.
+    assert len(semantic_manifest.metrics) == 2
 
     metric = next(
         (m for m in semantic_manifest.metrics if m.name == "test_conversion_metric_with_custom_grain_window"), None

tests/transformations/test_e2e_measure_to_model_transformations.py

@@ -0,0 +1,226 @@
+from dbt_semantic_interfaces.implementations.elements.dimension import (
+    PydanticDimension,
+    PydanticDimensionTypeParams,
+)
+from dbt_semantic_interfaces.implementations.elements.entity import PydanticEntity
+from dbt_semantic_interfaces.implementations.elements.measure import PydanticMeasure
+from dbt_semantic_interfaces.implementations.metric import (
+    PydanticCumulativeTypeParams,
+    PydanticMetric,
+    PydanticMetricInput,
+    PydanticMetricInputMeasure,
+    PydanticMetricTimeWindow,
+    PydanticMetricTypeParams,
+)
+from dbt_semantic_interfaces.implementations.node_relation import PydanticNodeRelation
+from dbt_semantic_interfaces.implementations.project_configuration import (
+    PydanticProjectConfiguration,
+)
+from dbt_semantic_interfaces.implementations.semantic_manifest import (
+    PydanticSemanticManifest,
+)
+from dbt_semantic_interfaces.implementations.semantic_model import (
+    PydanticSemanticModel,
+    PydanticSemanticModelDefaults,
+)
+from dbt_semantic_interfaces.transformations.semantic_manifest_transformer import (
+    PydanticSemanticManifestTransformer,
+)
+from dbt_semantic_interfaces.type_enums import (
+    AggregationType,
+    DimensionType,
+    EntityType,
+    MetricType,
+    TimeGranularity,
+)
+from dbt_semantic_interfaces.validations.semantic_manifest_validator import (
+    SemanticManifestValidator,
+)
+
+
+def _project_config() -> PydanticProjectConfiguration:
+    return PydanticProjectConfiguration()
+
+
+def _build_semantic_model_with_creatable_measure(
+    sm_name: str,
+    time_dim_name: str = "ds",
+) -> PydanticSemanticModel:
+    return PydanticSemanticModel(
+        name=sm_name,
+        defaults=PydanticSemanticModelDefaults(agg_time_dimension=time_dim_name),
+        node_relation=PydanticNodeRelation(alias=sm_name, schema_name="schema"),
+        entities=[
+            PydanticEntity(
+                name="user",
+                type=EntityType.PRIMARY,
+                expr="user_id",
+            ),
+        ],
+        dimensions=[
+            PydanticDimension(
+                name="ds",
+                type=DimensionType.TIME,
+                type_params=PydanticDimensionTypeParams(time_granularity=TimeGranularity.DAY),
+            ),
+            PydanticDimension(
+                name="created_at",
+                type=DimensionType.TIME,
+                type_params=PydanticDimensionTypeParams(time_granularity=TimeGranularity.DAY),
+            ),
+            PydanticDimension(
+                name="ds_partitioned",
+                type=DimensionType.TIME,
+                is_partition=True,
+                type_params=PydanticDimensionTypeParams(time_granularity=TimeGranularity.DAY),
+            ),
+            PydanticDimension(
+                name="home_state",
+                type=DimensionType.CATEGORICAL,
+            ),
+            PydanticDimension(
+                name="last_profile_edit_ts",
+                type=DimensionType.TIME,
+                type_params=PydanticDimensionTypeParams(time_granularity=TimeGranularity.MILLISECOND),
+            ),
+            PydanticDimension(
+                name="bio_added_ts",
+                type=DimensionType.TIME,
+                type_params=PydanticDimensionTypeParams(time_granularity=TimeGranularity.SECOND),
+            ),
+            PydanticDimension(
+                name="last_login_ts",
+                type=DimensionType.TIME,
+                type_params=PydanticDimensionTypeParams(time_granularity=TimeGranularity.MINUTE),
+            ),
+            PydanticDimension(
+                name="archived_at",
+                type=DimensionType.TIME,
+                type_params=PydanticDimensionTypeParams(time_granularity=TimeGranularity.HOUR),
+            ),
+        ],
+        measures=[
+            PydanticMeasure(
+                name="archived_users",
+                agg=AggregationType.SUM,
+                expr="1",
+                create_metric=True,
+            )
+        ],
+    )
+
+
+def test_e2e_measure_create_metric_then_cumulative_uses_metric_input() -> None:
+    """End-to-end: measure create_metric=True, cumulative references created metric by name."""
+    sm = _build_semantic_model_with_creatable_measure("sm", time_dim_name="ds")
+
+    metrics = [
+        PydanticMetric(
+            name="subdaily_cumulative_window_metric",
+            type=MetricType.CUMULATIVE,
+            description="m1_cumulative_1 description",
+            type_params=PydanticMetricTypeParams(
+                measure=PydanticMetricInputMeasure(name="archived_users"),
+                cumulative_type_params=PydanticCumulativeTypeParams(
+                    window=PydanticMetricTimeWindow(count=3, granularity="hour"),
+                ),
+            ),
+        ),
+        PydanticMetric(
+            name="subdaily_cumulative_grain_to_date_metric",
+            type=MetricType.CUMULATIVE,
+            description="m1_cumulative_2 description",
+            type_params=PydanticMetricTypeParams(
+                measure=PydanticMetricInputMeasure(name="archived_users"),
+                cumulative_type_params=PydanticCumulativeTypeParams(
+                    grain_to_date="hour",
+                ),
+            ),
+        ),
+        PydanticMetric(
+            name="subdaily_offset_window_metric",
+            type=MetricType.DERIVED,
+            description="archived_users_offset_window description",
+            type_params=PydanticMetricTypeParams(
+                expr="archived_users",
+                metrics=[
+                    PydanticMetricInput(
+                        name="archived_users",
+                        offset_window=PydanticMetricTimeWindow(count=1, granularity="hour"),
+                    )
+                ],
+            ),
+        ),
+        PydanticMetric(
+            name="subdaily_offset_grain_to_date_metric",
+            type=MetricType.DERIVED,
+            description="offset grain to date metric with a sub-daily agg time dim",
+            type_params=PydanticMetricTypeParams(
+                expr="archived_users",
+                metrics=[
+                    PydanticMetricInput(
+                        name="archived_users",
+                        offset_to_grain="hour",
+                    )
+                ],
+            ),
+        ),
+        PydanticMetric(
+            name="subdaily_join_to_time_spine_metric",
+            type=MetricType.SIMPLE,
+            description="simple metric with sub-daily agg time dim that joins to time spine",
+            type_params=PydanticMetricTypeParams(
+                measure=PydanticMetricInputMeasure(
+                    name="archived_users",
+                    join_to_timespine=True,
+                ),
+            ),
+        ),
+        PydanticMetric(
+            name="simple_subdaily_metric_default_day",
+            type=MetricType.SIMPLE,
+            description="simple metric with sub-daily agg time dim that doesn't specify default granularity",
+            type_params=PydanticMetricTypeParams(
+                measure=PydanticMetricInputMeasure(
+                    name="archived_users",
+                ),
+            ),
+        ),
+        PydanticMetric(
+            name="simple_subdaily_metric_default_hour",
+            type=MetricType.SIMPLE,
+            description="simple metric with sub-daily agg time dim that has an explicit default granularity",
+            type_params=PydanticMetricTypeParams(
+                measure=PydanticMetricInputMeasure(
+                    name="archived_users",
+                ),
+            ),
+            time_granularity="hour",
+        ),
+        PydanticMetric(
+            name="archived_users_join_to_time_spine",
+            type=MetricType.SIMPLE,
+            description="subdaily metric joining to time spine",
+            type_params=PydanticMetricTypeParams(
+                measure=PydanticMetricInputMeasure(
+                    name="archived_users",
+                    join_to_timespine=True,
+                ),
+            ),
+        ),
+    ]
+
+    manifest = PydanticSemanticManifest(
+        semantic_models=[sm],
+        metrics=metrics,
+        project_configuration=_project_config(),
+    )
+
+    transformed = PydanticSemanticManifestTransformer.transform(model=manifest)
+
+    model_validator = SemanticManifestValidator[PydanticSemanticManifest]()
+    model_validator.checked_validations(transformed)
+
+    # Expect exactly 1 new metric - the proxy simple metric created for the measure
+    assert len(transformed.metrics) == len(metrics) + 1
+    assert any(m for m in transformed.metrics if m.type == MetricType.SIMPLE and m.name == "archived_users")