diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index dd956c8c23..75a9e0eb26 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -277,9 +277,8 @@ Data source IDs can be found by hovering over data in charts. For the ``GET /api/v3_0/sensors//data`` endpoint specifically, source filtering supports: - ``source``: filter by data source ID -- ``account``: filter by the account ID linked to data sources - -Filtering that endpoint by source type is currently not supported. +- ``source-account``: filter by the account ID linked to data sources +- ``source-type``: filter by the type of data source (e.g. 'forecaster' or 'scheduler') .. _units: diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 106b5f72a0..612323d65e 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -12,6 +12,7 @@ New features * Improve UX after deleting a child asset through the UI [see `PR #2119 `_] * Improve source filtering in the sensor data GET endpoint by exposing the documented query parameters in Swagger and allowing filtering by the account linked to data sources [see `PR #2083 `_] * New ``GET /api/v3_0/sources`` endpoint to list accessible data sources and defined types, with ``only_latest=true`` by default to return only the most recent version per source [see `PR #2126 `_] +* Add support for filtering sensor data GET requests by ``source-type`` on ``/api/v3_0/sensors//data`` [see `PR #2127 `_] Infrastructure / Support ---------------------- diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 1cbed9dad2..1d77f694c4 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -168,12 +168,22 @@ class GetSensorDataFilterSchemaMixin: ), ) account = AccountIdField( + data_key="source-account", required=False, metadata=dict( description="Filter by the account linked to data sources.", example=3, ), ) + source_type = fields.Str( + data_key="source-type", + required=False, + validate=Length(min=1), + metadata=dict( + description="Filter by a specific data source type.", + example="forecaster", + ), + ) class GetSensorDataSchema(GetSensorDataFilterSchemaMixin, SensorDataDescriptionSchema): @@ -228,6 +238,7 @@ def load_data_and_make_response(sensor_data_description: dict) -> dict: resolution = sensor_data_description.get("resolution") source = sensor_data_description.get("source") account = sensor_data_description.get("account") + source_type = sensor_data_description.get("source_type") # Post-load configuration of event frequency if resolution is None: @@ -258,6 +269,7 @@ def load_data_and_make_response(sensor_data_description: dict) -> dict: horizons_at_most=horizons_at_most, source=source, source_account_ids=account.id if account else None, + source_types=[source_type] if source_type else None, beliefs_before=sensor_data_description.get("prior", None), one_deterministic_belief_per_event=True, resolution=resolution, diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index d8812392f9..5ed0413dde 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -648,7 +648,8 @@ def get_data(self, id: int, **sensor_data_description: dict): - "horizon" (read [the docs about belief timing](https://flexmeasures.readthedocs.io/latest/api/notation.html#tracking-the-recording-time-of-beliefs)) - "prior" (the belief timing docs also apply here) - "source" (filter by data source ID, read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources)) - - "account" (filter by the account ID linked to data sources) + - "source-account" (filter by the account ID linked to data sources) + - "source-type" (filter by data source type) An example query to fetch data for sensor with ID=1, for one hour starting June 7th 2021 at midnight, in 15 minute intervals, in m³/h: diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 274ed23401..873ecc182c 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -84,7 +84,7 @@ def test_get_sensor_data( None, None, ] - assert all(a == b for a, b in zip(values, expected)) + assert values == expected @pytest.mark.parametrize( @@ -142,7 +142,7 @@ def test_get_sensor_data_filtered_by_source_account( "duration": "PT1H20M", "horizon": "PT0H", "unit": "m³/h", - "account": source_user.account_id, + "source-account": source_user.account_id, "resolution": "PT20M", } response = client.get( @@ -166,6 +166,91 @@ def test_get_sensor_data_filtered_by_source_account( assert all(a == b for a, b in zip(values, expected)) +@pytest.mark.parametrize( + "source_type, expected", + [ + ( + "user", + [ + sum(GAS_MEASUREMENTS_10MIN[0:2]) / 2, # 91.5 + GAS_MEASUREMENTS_10MIN[2], # 92.1 + None, + None, + ], + ), + ( + "demo script", + [ + GAS_MEASUREMENTS_10MIN[0], # 91.3 + GAS_MEASUREMENTS_10MIN[2], # 92.1 + None, + None, + ], + ), + ("scheduler", [None, None, None, None]), + ], +) +@pytest.mark.parametrize( + "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True +) +def test_get_sensor_data_filtered_by_source_type( + client, + setup_api_test_data: dict[str, Sensor], + requesting_user, + source_type, + expected, +): + """Check that GET /sensors//data can filter by source type.""" + sensor = setup_api_test_data["some gas sensor"] + message = { + "start": "2021-05-02T00:00:00+02:00", + "duration": "PT1H20M", + "horizon": "PT0H", + "unit": "m³/h", + "source-type": source_type, + "resolution": "PT20M", + } + response = client.get( + url_for("SensorAPI:get_data", id=sensor.id), + query_string=message, + ) + print("Server responded with:\n%s" % response.json) + assert response.status_code == 200 + assert response.json["values"] == expected + + +@pytest.mark.parametrize( + "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True +) +def test_get_sensor_data_rejects_empty_source_type( + client, + setup_api_test_data: dict[str, Sensor], + requesting_user, +): + """Check that GET /sensors//data rejects an empty source-type filter.""" + sensor = setup_api_test_data["some gas sensor"] + message = { + "start": "2021-05-02T00:00:00+02:00", + "duration": "PT1H20M", + "horizon": "PT0H", + "unit": "m³/h", + "source-type": "", + "resolution": "PT20M", + } + response = client.get( + url_for("SensorAPI:get_data", id=sensor.id), + query_string=message, + ) + print("Server responded with:\n%s" % response.json) + assert response.status_code == 422 + assert ( + "Shorter than minimum length 1." + in response.json["message"]["combined_sensor_data_description"]["source-type"][ + 0 + ] + ) + + @pytest.mark.parametrize( "requesting_user, status_code", [ diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 1846f04756..a40a7bbcb9 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -365,7 +365,7 @@ }, "get": { "summary": "Get sensor data", - "description": "The unit has to be convertible from the sensor's unit - e.g. you ask for kW, and the sensor's unit is MW.\n\nOptional parameters:\n\n- \"resolution\" (read [the docs about frequency and resolutions](https://flexmeasures.readthedocs.io/latest/api/notation.html#frequency-and-resolution))\n- \"horizon\" (read [the docs about belief timing](https://flexmeasures.readthedocs.io/latest/api/notation.html#tracking-the-recording-time-of-beliefs))\n- \"prior\" (the belief timing docs also apply here)\n- \"source\" (filter by data source ID, read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources))\n- \"account\" (filter by the account ID linked to data sources)\n\nAn example query to fetch data for sensor with ID=1, for one hour starting June 7th 2021 at midnight, in 15 minute intervals, in m\u00b3/h:\n\n ?start=2021-06-07T00:00:00+02:00&duration=PT1H&resolution=PT15M&unit=m\u00b3/h\n\n(you will probably need to escape the + in the timezone offset, depending on your HTTP client, and other characters like here in the unit, as well).\n\n > Note: This endpoint also accepts the query parameters as part of the JSON body. That is not conform to REST architecture, but it is easier for some developers.\n", + "description": "The unit has to be convertible from the sensor's unit - e.g. you ask for kW, and the sensor's unit is MW.\n\nOptional parameters:\n\n- \"resolution\" (read [the docs about frequency and resolutions](https://flexmeasures.readthedocs.io/latest/api/notation.html#frequency-and-resolution))\n- \"horizon\" (read [the docs about belief timing](https://flexmeasures.readthedocs.io/latest/api/notation.html#tracking-the-recording-time-of-beliefs))\n- \"prior\" (the belief timing docs also apply here)\n- \"source\" (filter by data source ID, read [the docs about sources](https://flexmeasures.readthedocs.io/latest/api/notation.html#sources))\n- \"source-account\" (filter by the account ID linked to data sources)\n- \"source-type\" (filter by data source type)\n\nAn example query to fetch data for sensor with ID=1, for one hour starting June 7th 2021 at midnight, in 15 minute intervals, in m\u00b3/h:\n\n ?start=2021-06-07T00:00:00+02:00&duration=PT1H&resolution=PT15M&unit=m\u00b3/h\n\n(you will probably need to escape the + in the timezone offset, depending on your HTTP client, and other characters like here in the unit, as well).\n\n > Note: This endpoint also accepts the query parameters as part of the JSON body. That is not conform to REST architecture, but it is easier for some developers.\n", "security": [ { "ApiKeyAuth": [] @@ -457,13 +457,24 @@ }, { "in": "query", - "name": "account", + "name": "source-account", "description": "Filter by the account linked to data sources.", "schema": { "type": "integer", "example": 3 }, "required": false + }, + { + "in": "query", + "name": "source-type", + "description": "Filter by a specific data source type.", + "schema": { + "type": "string", + "minLength": 1, + "example": "forecaster" + }, + "required": false } ], "responses": {