diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 9e703ec80f..c13c844c76 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -21,7 +21,7 @@ Bugfixes v1.0.0 | August 14, 2026 ============================ -.. warning:: As of this release we standardize asynchronous job responses to use the ``job`` field and return HTTP ``202 Accepted`` while a background job is queued or running. See :ref:`api_background_jobs` for the response format and polling flow. Legacy response fields such as ``schedule`` and ``forecast`` will be deprecated; clients should migrate to ``job`` (see the Infrastructure / Support section below for migration details). +.. warning:: As of this release we standardize asynchronous job responses to use the ``job`` field and return HTTP ``202 Accepted`` while a background job is queued or running. See :ref:`api_background_jobs` for the response format and polling flow. Legacy response fields such as ``schedule`` and ``forecast`` will be deprecated; clients should migrate to ``job`` (see the Infrastructure / Support section below for migration details). To receive legacy status codes (e.g. for older clients), hosts can use :ref:`legacy-schedule-client-config`. .. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``). @@ -87,7 +87,7 @@ Infrastructure / Support ------------------------- * Support storing encrypted connection secrets on organisations and assets, including utility functions, encryption key configuration, CLI commands to set and delete secrets, and UI tables that show stored secret names and optional expiration times without exposing their values [see `PR #2236 `_] -* Standardize job-trigger API responses to return ``202 Accepted`` and a canonical ``job`` field; legacy response fields such as ``schedule`` and ``forecast`` are preserved for backward-compatibility but marked deprecated with migration guidance in :ref:`api_background_jobs` [see `PR #2224 `_]. +* Standardize job-trigger API responses to return ``202 Accepted`` and a canonical ``job`` field, and likewise return ``202 Accepted`` when polling a schedule whose job has not finished yet; legacy response fields such as ``schedule`` and ``forecast`` are preserved for backward-compatibility but marked deprecated with migration guidance in :ref:`api_background_jobs`. Hosts still serving clients that expect the previous status codes can opt individual assets back in, see :ref:`legacy-schedule-client-config` [see `PR #2224 `_ and `PR #2429 `_]. * Warn on startup when ``TRUSTED_HOSTS`` is unset, as that lets clients poison the URLs FlexMeasures generates, such as password reset links; the setting can now also be given as a comma-separated environment variable, and the ``development`` environment trusts loopback hosts by default (so reaching a development server by its LAN address or through a tunnel now means listing that host) [see `PR #2389 `_] * Upgraded dependencies [see `PR #1485 `_, `PR #2215 `_, `PR #2243 `_, `PR #2348 `_ and `PR #2388 `_] * Add a ``FLEXMEASURES_SENTRY_DAILY_RATE_LIMIT`` setting for spreading a host's Sentry error allowance across the month with a fail-open daily Redis counter, and send the startup error about the database schema not being at the Alembic head revision to Sentry at most once per UTC calendar day per pair of current and expected revisions (it is still logged in full on every start) [see `PR #2366 `_] diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 6ea3081a67..68e03bb585 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -1024,6 +1024,30 @@ If ``False``, the API transparently follows the fallback job and returns the fal Default: ``False`` + +.. _legacy-schedule-client-config: + +FLEXMEASURES_LEGACY_SCHEDULEACCEPTED_STATUS_MAX_INCOMPATIBLE_CLIENT_VERSION +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Backwards-compatibility switch for scheduling-related endpoints in API v3. + +Mapping of version-valued asset attribute names to the maximum incompatible version for clients that still expect schedule trigger requests to return ``HTTP status 200 (OK)`` and unfinished schedule requests to return ``HTTP status 400`` with a message about the scheduling job "waiting to be processed". +For each configured attribute, FlexMeasures checks the scheduled asset itself, its parent asset, and its grandparent asset. +If any of these attributes contains its configured maximum version or a lower version, the client receives the legacy schedule responses. +When empty, all clients receive the standard ``202 Accepted`` response for accepted trigger requests and unfinished schedule requests. + +For example: + +.. code-block:: python + + FLEXMEASURES_LEGACY_SCHEDULEACCEPTED_STATUS_MAX_INCOMPATIBLE_CLIENT_VERSION = { + "v2g-liberty-version": "0.9.1", + } + +Default: ``{}`` + + .. _reporters-config: Reporters diff --git a/flexmeasures/api/common/responses.py b/flexmeasures/api/common/responses.py index 241d68ed7b..d4abe8a014 100644 --- a/flexmeasures/api/common/responses.py +++ b/flexmeasures/api/common/responses.py @@ -387,6 +387,7 @@ def request_accepted_for_processing( message: str = "Request has been accepted for processing.", legacy_key: str | None = None, job_results_url: str | None = None, + status_code: int = 202, ) -> ResponseTuple: """ Standard 202 response when a background job is accepted. @@ -414,7 +415,7 @@ def request_accepted_for_processing( # keep legacy key for backwards compatibility; not (yet) deprecated, see docstring resp[legacy_key] = job_id - return resp, 202 + return resp, status_code def request_too_large(message: str) -> ResponseTuple: diff --git a/flexmeasures/api/common/utils/api_utils.py b/flexmeasures/api/common/utils/api_utils.py index 31c879634a..aa99a3326e 100644 --- a/flexmeasures/api/common/utils/api_utils.py +++ b/flexmeasures/api/common/utils/api_utils.py @@ -1,8 +1,10 @@ from __future__ import annotations +from collections.abc import Mapping from copy import deepcopy import json import re +from packaging.version import InvalidVersion, Version from timely_beliefs.beliefs.classes import BeliefsDataFrame from timely_beliefs.sensors.func_store import knowledge_horizons from typing import Sequence @@ -62,6 +64,57 @@ def upsample_values( return value_groups +def use_legacy_schedule_accepted_status(asset: GenericAsset) -> bool: + version_limits = current_app.config.get( + "FLEXMEASURES_LEGACY_SCHEDULEACCEPTED_STATUS_MAX_INCOMPATIBLE_CLIENT_VERSION", + {}, + ) + if not isinstance(version_limits, Mapping): + current_app.logger.warning( + "Invalid FLEXMEASURES_LEGACY_SCHEDULEACCEPTED_STATUS_MAX_INCOMPATIBLE_" + "CLIENT_VERSION %r: expected a mapping of asset attribute names to " + "maximum incompatible client versions. Ignoring compatibility setting.", + version_limits, + ) + return False + + for version_attribute, max_version in version_limits.items(): + client_version, attribute_asset = _get_asset_attribute_from_nearby_hierarchy( + asset, version_attribute + ) + if client_version is None or attribute_asset is None: + continue + try: + if Version(str(client_version)) <= Version(str(max_version)): + return True + except InvalidVersion: + current_app.logger.warning( + "Ignoring invalid schedule client version %r or maximum incompatible " + "version %r for attribute %r on asset %s.", + client_version, + max_version, + version_attribute, + attribute_asset.id, + ) + return False + + +def _get_asset_attribute_from_nearby_hierarchy( + asset: GenericAsset, attribute: str, max_parent_depth: int = 2 +) -> tuple[object | None, GenericAsset | None]: + current_asset = asset + for _ in range(max_parent_depth + 1): + # A null or empty value means the attribute is not set here, so keep looking up the hierarchy. + # Stopping on mere key presence would let such a value on a device shadow a version set on its site. + value = (current_asset.attributes or {}).get(attribute) + if value: + return value, current_asset + if current_asset.parent_asset is None: + break + current_asset = current_asset.parent_asset + return None, None + + def unique_ever_seen(iterable: Sequence, selector: Sequence): """ Return unique iterable elements with corresponding lists of selector elements, preserving order. diff --git a/flexmeasures/api/v3_0/assets.py b/flexmeasures/api/v3_0/assets.py index 0668cde606..efaf38b159 100644 --- a/flexmeasures/api/v3_0/assets.py +++ b/flexmeasures/api/v3_0/assets.py @@ -72,6 +72,7 @@ from flexmeasures.api.common.utils.api_utils import ( get_accessible_accounts, copy_asset, + use_legacy_schedule_accepted_status, ) from flexmeasures.api.common.responses import ( unprocessable_entity, @@ -1794,6 +1795,7 @@ def trigger_schedule( return request_accepted_for_processing( job.id, legacy_key="schedule", + status_code=200 if use_legacy_schedule_accepted_status(asset) else 202, ) @route("//kpis", methods=["GET"]) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 204a9f5017..d4032239e9 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -41,7 +41,10 @@ from flexmeasures.api.common.schemas.sensors import SensorId # noqa F401 from flexmeasures.api.common.schemas.users import AccountIdField from flexmeasures.api.common.rate_limiting import limit_triggers -from flexmeasures.api.common.utils.api_utils import process_sensor_data_ingestion +from flexmeasures.api.common.utils.api_utils import ( + process_sensor_data_ingestion, + use_legacy_schedule_accepted_status, +) from flexmeasures.data.services.utils import job_status_description from flexmeasures.api.common.utils.deprecation_utils import ( _add_headers as add_deprecation_header, @@ -1090,6 +1093,11 @@ def trigger_schedule( job_results_url=url_for( "SensorAPI:get_schedule", id=sensor.id, uuid=job.id ), + status_code=( + 200 + if use_legacy_schedule_accepted_status(sensor.generic_asset) + else 202 + ), ) # mark endpoint as deprecated @@ -1326,21 +1334,20 @@ def get_schedule( # noqa: C901 elif job.is_failed: return unknown_schedule(job_status_description(job, scheduler_info_msg)) else: - if current_app.config.get("FLEXMEASURES_API_SUNSET_ACTIVE"): - job_status = job.get_status() - job_status_name = ( - job_status.upper() - if isinstance(job_status, str) - else job_status.name - ) - return ( - dict( - status=job_status_name, - message=job_status_description(job, scheduler_info_msg), - ), - 202, - ) - return unknown_schedule(job_status_description(job, scheduler_info_msg)) + job_status = job.get_status() + job_status_name = ( + job_status.upper() if isinstance(job_status, str) else job_status.name + ) + response = dict( + status=job_status_name, + message=job_status_description(job, scheduler_info_msg), + ) + if use_legacy_schedule_accepted_status(sensor.generic_asset): + return response, 400 + return ( + response, + 202, + ) schedule_start = job.kwargs["start"] data_source = get_data_source_for_job(job) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 2e706c2250..8e1a6cbab9 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -12,16 +12,18 @@ unknown_schedule, unrecognized_event, ) +from flexmeasures.api.common.utils.api_utils import use_legacy_schedule_accepted_status from flexmeasures.api.tests.utils import check_deprecation from flexmeasures.api.v3_0.tests.utils import ( get_sensor_by_name, message_for_trigger_schedule, ) from flexmeasures.data.models.data_sources import DataSource +from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.models.time_series import Sensor -from flexmeasures.utils.job_utils import work_on_rq from flexmeasures.data.services.scheduling import handle_scheduling_exception from flexmeasures.tests.utils import get_test_sensor +from flexmeasures.utils.job_utils import work_on_rq from flexmeasures.utils.unit_utils import ur @@ -343,18 +345,149 @@ def test_trigger_and_get_schedule_with_unknown_prices( assert "prices unknown" in get_schedule_response.json["message"].lower() +@pytest.mark.parametrize("version_attribute_level", ["asset", "parent", "grandparent"]) +def test_legacy_schedule_accepted_status_checks_nearby_asset_hierarchy( + app, + add_battery_assets, + monkeypatch, + version_attribute_level, +): + battery = add_battery_assets["Test battery"] + building = add_battery_assets["Test building"] + version_attribute = "flexmeasures-client-version" + monkeypatch.setitem( + app.config, + "FLEXMEASURES_LEGACY_SCHEDULEACCEPTED_STATUS_MAX_INCOMPATIBLE_CLIENT_VERSION", + {"other-client-version": "1.0.0", version_attribute: "0.9.1"}, + ) + battery.attributes = { + key: value + for key, value in (battery.attributes or {}).items() + if key != version_attribute + } + building.attributes = { + key: value + for key, value in (building.attributes or {}).items() + if key != version_attribute + } + + if version_attribute_level == "asset": + battery.attributes = {**(battery.attributes or {}), version_attribute: "0.7.0"} + elif version_attribute_level == "parent": + building.attributes = { + **(building.attributes or {}), + version_attribute: "0.7.0", + } + else: + site = GenericAsset( + name="schedule client version site", + generic_asset_type=building.generic_asset_type, + owner=building.owner, + attributes={version_attribute: "0.7.0"}, + ) + monkeypatch.setattr(building, "parent_asset", site) + + assert use_legacy_schedule_accepted_status(battery) + + +@pytest.mark.parametrize("shadowing_value", [None, ""]) +def test_legacy_schedule_accepted_status_looks_past_empty_attribute_value( + app, + add_battery_assets, + monkeypatch, + shadowing_value, +): + """A null or empty value on the asset should not hide a version set on its parent.""" + battery = add_battery_assets["Test battery"] + building = add_battery_assets["Test building"] + version_attribute = "flexmeasures-client-version" + monkeypatch.setitem( + app.config, + "FLEXMEASURES_LEGACY_SCHEDULEACCEPTED_STATUS_MAX_INCOMPATIBLE_CLIENT_VERSION", + {version_attribute: "0.9.1"}, + ) + battery.attributes = { + **(battery.attributes or {}), + version_attribute: shadowing_value, + } + building.attributes = {**(building.attributes or {}), version_attribute: "0.7.0"} + + assert use_legacy_schedule_accepted_status(battery) + + +def test_legacy_schedule_accepted_status_ignores_non_mapping_config( + app, + add_battery_assets, + monkeypatch, + caplog, +): + monkeypatch.setitem( + app.config, + "FLEXMEASURES_LEGACY_SCHEDULEACCEPTED_STATUS_MAX_INCOMPATIBLE_CLIENT_VERSION", + "0.9.1", + ) + + assert not use_legacy_schedule_accepted_status(add_battery_assets["Test battery"]) + assert "expected a mapping of asset attribute names" in caplog.text + + +@pytest.mark.parametrize("trigger_endpoint", ["sensor", "asset"]) @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) -def test_get_schedule_unfinished_job_returns_202_when_sunset_active( +def test_trigger_schedule_returns_200_for_legacy_schedule_accepted_status( app, + db, add_battery_assets, keep_scheduling_queue_empty, + monkeypatch, requesting_user, + trigger_endpoint, ): sensor = add_battery_assets["Test battery"].sensors[0] - original_sunset_active = app.config.get("FLEXMEASURES_API_SUNSET_ACTIVE") - app.config["FLEXMEASURES_API_SUNSET_ACTIVE"] = True + version_attribute = "flexmeasures-client-version" + monkeypatch.setitem( + app.config, + "FLEXMEASURES_LEGACY_SCHEDULEACCEPTED_STATUS_MAX_INCOMPATIBLE_CLIENT_VERSION", + {version_attribute: "0.9.1"}, + ) + sensor.generic_asset.attributes = { + **(sensor.generic_asset.attributes or {}), + version_attribute: "0.7.0", + } + db.session.commit() + + message = message_for_trigger_schedule() + if trigger_endpoint == "asset": + message["flex-model"] = [{**message["flex-model"], "sensor": sensor.id}] + url = url_for("AssetAPI:trigger_schedule", id=sensor.generic_asset.id) + else: + url = url_for("SensorAPI:trigger_schedule", id=sensor.id) + + with app.test_client() as client: + trigger_schedule_response = client.post(url, json=message) + + assert trigger_schedule_response.status_code == 200 + assert ( + trigger_schedule_response.json["job"] + == trigger_schedule_response.json["schedule"] + ) + assert len(app.queues["scheduling"]) == 1 + + +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_get_schedule_unfinished_job_returns_202_by_default( + app, + db, + add_battery_assets, + keep_scheduling_queue_empty, + monkeypatch, + requesting_user, +): + sensor = add_battery_assets["Test battery"].sensors[0] + monkeypatch.setitem(app.config, "FLEXMEASURES_API_SUNSET_ACTIVE", False) with app.test_client() as client: trigger_schedule_response = client.post( @@ -372,16 +505,47 @@ def test_get_schedule_unfinished_job_returns_202_when_sunset_active( assert get_schedule_response.json["status"] in {"QUEUED", "STARTED", "DEFERRED"} assert "message" in get_schedule_response.json - app.config["FLEXMEASURES_API_SUNSET_ACTIVE"] = False + version_attribute = "flexmeasures-client-version" + monkeypatch.setitem( + app.config, + "FLEXMEASURES_LEGACY_SCHEDULEACCEPTED_STATUS_MAX_INCOMPATIBLE_CLIENT_VERSION", + {version_attribute: "0.9.1"}, + ) + sensor.generic_asset.attributes = { + **(sensor.generic_asset.attributes or {}), + version_attribute: "0.9.1", + } + db.session.commit() + with app.test_client() as client: - get_schedule_response_old = client.get( + get_schedule_response_legacy_client = client.get( url_for("SensorAPI:get_schedule", id=sensor.id, uuid=job_id), ) - app.config["FLEXMEASURES_API_SUNSET_ACTIVE"] = original_sunset_active + assert get_schedule_response_legacy_client.status_code == 400 + # Legacy flexmeasures-client releases retry HTTP 400 responses whose + # message contains this exact, long-standing substring. + assert ( + "Scheduling job waiting" in get_schedule_response_legacy_client.json["message"] + ) + assert get_schedule_response_legacy_client.json["status"] in { + "QUEUED", + "STARTED", + "DEFERRED", + } + + sensor.generic_asset.attributes = { + **(sensor.generic_asset.attributes or {}), + version_attribute: "0.9.2", + } + db.session.commit() + + with app.test_client() as client: + get_schedule_response_compatible_client = client.get( + url_for("SensorAPI:get_schedule", id=sensor.id, uuid=job_id), + ) - assert get_schedule_response_old.status_code == 400 - assert get_schedule_response_old.json["status"] == unknown_schedule()[0]["status"] + assert get_schedule_response_compatible_client.status_code == 202 @pytest.mark.parametrize( diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 7e3ab548a7..097fea4222 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -7,7 +7,7 @@ }, "termsOfService": null, "title": "FlexMeasures", - "version": "1.0.0" + "version": "1.0.0rc5" }, "externalDocs": { "description": "FlexMeasures runs on the open source FlexMeasures technology. Read the docs here.", @@ -7278,4 +7278,4 @@ } } } -} +} \ No newline at end of file diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index dc7cae05c7..01318df824 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -210,6 +210,9 @@ class Config(object): JSON_SORT_KEYS = False FLEXMEASURES_FALLBACK_REDIRECT: bool = False + FLEXMEASURES_LEGACY_SCHEDULEACCEPTED_STATUS_MAX_INCOMPATIBLE_CLIENT_VERSION: dict[ + str, str + ] = {} # Custom sunset switches FLEXMEASURES_API_SUNSET_ACTIVE: bool = (