Feat: Cleanup stale assets in graphs and configs

documentation/changelog.rst

@@ -42,6 +42,7 @@ Infrastructure / Support
 * Speed up sensor stats, especially potent when viewing sensors stats over a large sensor history [`PR #2173 <https://www.github.com/FlexMeasures/flexmeasures/pull/2173>`_]
 * Various smaller fixes in documenting scheduling endpoints and flex-model fields [`PR #2122 <https://www.github.com/FlexMeasures/flexmeasures/pull/2122>`_]
 * Add cross-cutting Copilot instruction files to ``.github/instructions/`` covering atomic commits, changelog format, docstrings, error handling, Marshmallow schemas, pre-commit hooks, testing, timezone awareness, and UI terminology [see `PR #2198 <https://www.github.com/FlexMeasures/flexmeasures/pull/2198>`_]
+* Ignore stale asset references in ``sensors_to_show`` so chart endpoints and the graph editor remain usable, and automatically prune those references when deleting assets [see `PR #2210 <https://www.github.com/FlexMeasures/flexmeasures/pull/2210>`_]
 
 Bugfixes
 -----------

flexmeasures/api/v3_0/tests/test_assets_api_fresh_db.py

@@ -1,10 +1,12 @@
+import json
+
 import pytest
 from flask import url_for
 from sqlalchemy import select
 
 from flexmeasures.api.tests.utils import AccountContext
 from flexmeasures.data.models.generic_assets import GenericAsset
-from flexmeasures.api.v3_0.tests.utils import get_asset_post_data
+from flexmeasures.api.v3_0.tests.utils import get_asset_post_data, check_audit_log_event
 
 
 @pytest.mark.parametrize(
@@ -100,3 +102,55 @@ def test_patch_asset_accepts_flex_context_object(
     ).scalar_one_or_none()
     assert updated_asset is not None
     assert updated_asset.flex_context["site-power-capacity"] == "1000 kW"
+
+
+@pytest.mark.parametrize("requesting_user", ["test_admin_user@seita.nl"], indirect=True)
+def test_delete_asset_cleans_stale_asset_references_in_sensors_to_show(
+    client, setup_api_fresh_test_data, requesting_user, fresh_db
+):
+    """Verify that deleting an asset cleans up stale asset references in other assets' sensors_to_show."""
+    db = fresh_db
+    deleted_asset = setup_api_fresh_test_data["some gas sensor"].generic_asset
+    deleted_asset_id = deleted_asset.id
+    deleted_asset_name = deleted_asset.name
+    referenced_sensor = setup_api_fresh_test_data["some temperature sensor"]
+
+    # Use a dedicated asset as the reference holder so deleting `deleted_asset`
+    # does not remove the object we assert on later.
+    referencing_asset = GenericAsset(
+        name="stale-asset-ref-holder",
+        generic_asset_type_id=deleted_asset.generic_asset_type_id,
+        account_id=requesting_user.account_id,
+    )
+    db.session.add(referencing_asset)
+    db.session.flush()
+
+    referencing_asset.sensors_to_show = [
+        {
+            "title": "Mixed graph",
+            "plots": [
+                {"sensor": referenced_sensor.id},
+                {"asset": deleted_asset_id, "flex-model": "soc-min"},
+            ],
+        }
+    ]
+    db.session.add(referencing_asset)
+    db.session.commit()
+
+    delete_asset_response = client.delete(
+        url_for("AssetAPI:delete", id=deleted_asset_id),
+    )
+    assert delete_asset_response.status_code == 204
+
+    updated_referencing_asset = db.session.get(GenericAsset, referencing_asset.id)
+    assert updated_referencing_asset is not None
+    assert str(deleted_asset_id) not in json.dumps(
+        updated_referencing_asset.sensors_to_show
+    )
+
+    check_audit_log_event(
+        db=db,
+        event=f"Removed asset reference '{deleted_asset_name}': {deleted_asset_id} from sensors-to-show (because asset has been deleted).",
+        user=requesting_user,
+        asset=updated_referencing_asset,
+    )

flexmeasures/data/schemas/generic_assets.py

@@ -5,6 +5,7 @@
 from http import HTTPStatus
 from typing import Any, TYPE_CHECKING
 
+from flask import current_app
 from flask import abort
 from marshmallow import validates, ValidationError, fields, validates_schema
 from marshmallow.validate import OneOf
@@ -196,7 +197,12 @@ def _validate_flex_config_field_is_valid_choice(
         if field_name in plot_config:
             value = plot_config[field_name]
             asset_id = plot_config.get("asset")
-            asset = GenericAssetIdField().deserialize(asset_id)
+            try:
+                asset = GenericAssetIdField().deserialize(asset_id)
+            except ValidationError:
+                if self.allow_missing_asset_flex_field:
+                    return
+                raise
 
             if asset is None:
                 raise ValidationError(f"Asset with ID {asset_id} does not exist.")
@@ -251,28 +257,72 @@ def flatten(cls, nested_list: list) -> list[int] | list[Sensor]:
         :param nested_list: A list containing sensor IDs, or dictionaries with `sensors` or `sensor` keys.
         :returns:           A unique list of sensor IDs, or a unique list of Sensors
         """
-        all_objects = []
+        all_objects: list[Any] = []
         for s in nested_list:
-            if isinstance(s, list):
-                all_objects.extend(s)
-            elif isinstance(s, int):
-                all_objects.append(s)
-            elif isinstance(s, dict):
-                if "plots" in s:
-                    for plot in s["plots"]:
-                        if "sensors" in plot:
-                            all_objects.extend(plot["sensors"])
-                        if "sensor" in plot:
-                            all_objects.append(plot["sensor"])
-                        if "asset" in plot:
-                            sensors, _ = extract_sensors_from_flex_config(plot)
-                            all_objects.extend(sensors)
-                elif "sensors" in s:
-                    all_objects.extend(s["sensors"])
-                elif "sensor" in s:
-                    all_objects.append(s["sensor"])
+            all_objects.extend(cls._flatten_entry(s))
         return list(dict.fromkeys(all_objects).keys())
 
+    @classmethod
+    def _flatten_entry(cls, entry: Any) -> list[Any]:
+        """Flatten one sensors_to_show entry into sensor IDs or Sensor objects."""
+        if isinstance(entry, list):
+            return entry
+        if isinstance(entry, int):
+            return [entry]
+        if isinstance(entry, dict):
+            return cls._flatten_dict_entry(entry)
+        return []
+
+    @classmethod
+    def _flatten_dict_entry(cls, entry: dict) -> list[Any]:
+        """Flatten one dictionary-form sensors_to_show entry."""
+        if "plots" in entry:
+            return cls._flatten_plots(entry["plots"])
+        if "sensors" in entry:
+            return entry["sensors"]
+        if "sensor" in entry:
+            return [entry["sensor"]]
+        return []
+
+    @classmethod
+    def _flatten_plots(cls, plots: list[dict]) -> list[Any]:
+        """Flatten a list of plot dictionaries to sensor IDs or Sensor objects."""
+        flattened: list[Any] = []
+        for plot in plots:
+            flattened.extend(cls._flatten_plot_entry(plot))
+        return flattened
+
+    @classmethod
+    def _flatten_plot_entry(cls, plot: dict) -> list[Any]:
+        """Flatten one plot dictionary to sensor IDs or Sensor objects."""
+        flattened: list[Any] = []
+
+        if "sensors" in plot:
+            flattened.extend(plot["sensors"])
+        if "sensor" in plot:
+            flattened.append(plot["sensor"])
+        if "asset" in plot:
+            flattened.extend(cls._flatten_asset_plot_entry(plot))
+
+        return flattened
+
+    @classmethod
+    def _flatten_asset_plot_entry(cls, plot: dict) -> list[Sensor]:
+        """Resolve flex-config sensor references for one asset plot.
+
+        Invalid stale references are skipped to keep chart rendering robust.
+        """
+        try:
+            sensors, _ = extract_sensors_from_flex_config(plot)
+        except ValidationError as err:
+            current_app.logger.warning(
+                "Skipping invalid asset plot reference %s while flattening sensors_to_show: %s",
+                plot,
+                err,
+            )
+            return []
+        return sensors
+
 
 class SensorKPIFieldSchema(ma.SQLAlchemySchema):
     title = fields.Str(required=True)