Feat: sensor data source type filter

documentation/api/notation.rst

@@ -277,9 +277,8 @@ Data source IDs can be found by hovering over data in charts.
 For the ``GET /api/v3_0/sensors/<id>/data`` endpoint specifically, source filtering supports:
 
 - ``source``: filter by data source ID
-- ``account``: filter by the account ID linked to data sources
-
-Filtering that endpoint by source type is currently not supported.
+- ``source-account``: filter by the account ID linked to data sources
+- ``source-type``: filter by the type of data source (e.g. 'forecaster' or 'scheduler')
 
 .. _units:
 

documentation/changelog.rst

@@ -12,6 +12,7 @@ New features
 * Improve UX after deleting a child asset through the UI [see `PR #2119 <https://www.github.com/FlexMeasures/flexmeasures/pull/2119>`_]
 * Improve source filtering in the sensor data GET endpoint by exposing the documented query parameters in Swagger and allowing filtering by the account linked to data sources [see `PR #2083 <https://www.github.com/FlexMeasures/flexmeasures/pull/2083>`_]
 * New ``GET /api/v3_0/sources`` endpoint to list accessible data sources and defined types, with ``only_latest=true`` by default to return only the most recent version per source [see `PR #2126 <https://www.github.com/FlexMeasures/flexmeasures/pull/2126>`_]
+* Add support for filtering sensor data GET requests by ``source-type`` on ``/api/v3_0/sensors/<id>/data`` [see `PR #2127 <https://www.github.com/FlexMeasures/flexmeasures/pull/2127>`_]
 
 Infrastructure / Support
 ----------------------

flexmeasures/api/common/schemas/sensor_data.py

@@ -168,12 +168,22 @@ class GetSensorDataFilterSchemaMixin:
         ),
     )
     account = AccountIdField(
+        data_key="source-account",
         required=False,
         metadata=dict(
             description="Filter by the account linked to data sources.",
             example=3,
         ),
     )
+    source_type = fields.Str(
+        data_key="source-type",
+        required=False,
+        validate=Length(min=1),
+        metadata=dict(
+            description="Filter by a specific data source type.",
+            example="forecaster",
+        ),
+    )
 
 
 class GetSensorDataSchema(GetSensorDataFilterSchemaMixin, SensorDataDescriptionSchema):
@@ -228,6 +238,7 @@ def load_data_and_make_response(sensor_data_description: dict) -> dict:
         resolution = sensor_data_description.get("resolution")
         source = sensor_data_description.get("source")
         account = sensor_data_description.get("account")
+        source_type = sensor_data_description.get("source_type")
 
         # Post-load configuration of event frequency
         if resolution is None:
@@ -258,6 +269,7 @@ def load_data_and_make_response(sensor_data_description: dict) -> dict:
                 horizons_at_most=horizons_at_most,
                 source=source,
                 source_account_ids=account.id if account else None,
+                source_types=[source_type] if source_type else None,
                 beliefs_before=sensor_data_description.get("prior", None),
                 one_deterministic_belief_per_event=True,
                 resolution=resolution,

flexmeasures/api/v3_0/sensors.py

@@ -648,7 +648,8 @@ def get_data(self, id: int, **sensor_data_description: dict):
             - "horizon" (read [the docs about belief timing](https://flexmeasures.readthedocs.io/latest/api/notation.html#tracking-the-recording-time-of-beliefs))
             - "prior" (the belief timing docs also apply here)
             - "source" (filter by data source ID, read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources))
-            - "account" (filter by the account ID linked to data sources)
+            - "source-account" (filter by the account ID linked to data sources)
+            - "source-type" (filter by data source type)
 
             An example query to fetch data for sensor with ID=1, for one hour starting June 7th 2021 at midnight, in 15 minute intervals, in m³/h:
 

flexmeasures/api/v3_0/tests/test_sensor_data.py

@@ -84,7 +84,7 @@ def test_get_sensor_data(
         None,
         None,
     ]
-    assert all(a == b for a, b in zip(values, expected))
+    assert values == expected
 
 
 @pytest.mark.parametrize(
@@ -142,7 +142,7 @@ def test_get_sensor_data_filtered_by_source_account(
         "duration": "PT1H20M",
         "horizon": "PT0H",
         "unit": "m³/h",
-        "account": source_user.account_id,
+        "source-account": source_user.account_id,
         "resolution": "PT20M",
     }
     response = client.get(
@@ -166,6 +166,91 @@ def test_get_sensor_data_filtered_by_source_account(
     assert all(a == b for a, b in zip(values, expected))
 
 
+@pytest.mark.parametrize(
+    "source_type, expected",
+    [
+        (
+            "user",
+            [
+                sum(GAS_MEASUREMENTS_10MIN[0:2]) / 2,  # 91.5
+                GAS_MEASUREMENTS_10MIN[2],  # 92.1
+                None,
+                None,
+            ],
+        ),
+        (
+            "demo script",
+            [
+                GAS_MEASUREMENTS_10MIN[0],  # 91.3
+                GAS_MEASUREMENTS_10MIN[2],  # 92.1
+                None,
+                None,
+            ],
+        ),
+        ("scheduler", [None, None, None, None]),
+    ],
+)
+@pytest.mark.parametrize(
+    "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True
+)
+def test_get_sensor_data_filtered_by_source_type(
+    client,
+    setup_api_test_data: dict[str, Sensor],
+    requesting_user,
+    source_type,
+    expected,
+):
+    """Check that GET /sensors/<id>/data can filter by source type."""
+    sensor = setup_api_test_data["some gas sensor"]
+    message = {
+        "start": "2021-05-02T00:00:00+02:00",
+        "duration": "PT1H20M",
+        "horizon": "PT0H",
+        "unit": "m³/h",
+        "source-type": source_type,
+        "resolution": "PT20M",
+    }
+    response = client.get(
+        url_for("SensorAPI:get_data", id=sensor.id),
+        query_string=message,
+    )
+    print("Server responded with:\n%s" % response.json)
+    assert response.status_code == 200
+    assert response.json["values"] == expected
+
+
+@pytest.mark.parametrize(
+    "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True
+)
+def test_get_sensor_data_rejects_empty_source_type(
+    client,
+    setup_api_test_data: dict[str, Sensor],
+    requesting_user,
+):
+    """Check that GET /sensors/<id>/data rejects an empty source-type filter."""
+    sensor = setup_api_test_data["some gas sensor"]
+    message = {
+        "start": "2021-05-02T00:00:00+02:00",
+        "duration": "PT1H20M",
+        "horizon": "PT0H",
+        "unit": "m³/h",
+        "source-type": "",
+        "resolution": "PT20M",
+    }
+    response = client.get(
+        url_for("SensorAPI:get_data", id=sensor.id),
+        query_string=message,
+    )
+    print("Server responded with:\n%s" % response.json)
+    assert response.status_code == 422
+    assert (
+        "Shorter than minimum length 1."
+        in response.json["message"]["combined_sensor_data_description"]["source-type"][
+            0
+        ]
+    )
+
+
 @pytest.mark.parametrize(
     "requesting_user, status_code",
     [

flexmeasures/ui/static/openapi-specs.json

@@ -365,7 +365,7 @@
       },
       "get": {
         "summary": "Get sensor data",
-        "description": "The unit has to be convertible from the sensor's unit - e.g. you ask for kW, and the sensor's unit is MW.\n\nOptional parameters:\n\n- \"resolution\" (read [the docs about frequency and resolutions](https://flexmeasures.readthedocs.io/latest/api/notation.html#frequency-and-resolution))\n- \"horizon\" (read [the docs about belief timing](https://flexmeasures.readthedocs.io/latest/api/notation.html#tracking-the-recording-time-of-beliefs))\n- \"prior\" (the belief timing docs also apply here)\n- \"source\" (filter by data source ID, read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources))\n- \"account\" (filter by the account ID linked to data sources)\n\nAn example query to fetch data for sensor with ID=1, for one hour starting June 7th 2021 at midnight, in 15 minute intervals, in m\u00b3/h:\n\n  ?start=2021-06-07T00:00:00+02:00&duration=PT1H&resolution=PT15M&unit=m\u00b3/h\n\n(you will probably need to escape the + in the timezone offset, depending on your HTTP client, and other characters like here in the unit, as well).\n\n > <strong>Note:</strong> This endpoint also accepts the query parameters as part of the JSON body. That is not conform to REST architecture, but it is easier for some developers.\n",
+        "description": "The unit has to be convertible from the sensor's unit - e.g. you ask for kW, and the sensor's unit is MW.\n\nOptional parameters:\n\n- \"resolution\" (read [the docs about frequency and resolutions](https://flexmeasures.readthedocs.io/latest/api/notation.html#frequency-and-resolution))\n- \"horizon\" (read [the docs about belief timing](https://flexmeasures.readthedocs.io/latest/api/notation.html#tracking-the-recording-time-of-beliefs))\n- \"prior\" (the belief timing docs also apply here)\n- \"source\" (filter by data source ID, read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources))\n- \"source-account\" (filter by the account ID linked to data sources)\n- \"source-type\" (filter by data source type)\n\nAn example query to fetch data for sensor with ID=1, for one hour starting June 7th 2021 at midnight, in 15 minute intervals, in m\u00b3/h:\n\n  ?start=2021-06-07T00:00:00+02:00&duration=PT1H&resolution=PT15M&unit=m\u00b3/h\n\n(you will probably need to escape the + in the timezone offset, depending on your HTTP client, and other characters like here in the unit, as well).\n\n > <strong>Note:</strong> This endpoint also accepts the query parameters as part of the JSON body. That is not conform to REST architecture, but it is easier for some developers.\n",
         "security": [
           {
             "ApiKeyAuth": []
@@ -457,13 +457,24 @@
           },
           {
             "in": "query",
-            "name": "account",
+            "name": "source-account",
             "description": "Filter by the account linked to data sources.",
             "schema": {
               "type": "integer",
               "example": 3
             },
             "required": false
+          },
+          {
+            "in": "query",
+            "name": "source-type",
+            "description": "Filter by a specific data source type.",
+            "schema": {
+              "type": "string",
+              "minLength": 1,
+              "example": "forecaster"
+            },
+            "required": false
           }
         ],
         "responses": {