diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 6e8d585933..63c75aabb1 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -7,6 +7,8 @@ FlexMeasures Changelog v1.0.0 | July XX, 2026 ============================ +.. warning:: Upgrading to this version requires running ``flexmeasures db upgrade`` (you can create a backup first with ``flexmeasures db-ops dump``). + New features ------------- * In the UI, asset and sensor lists can be filtered by ID prefix through API-backed search fields [see `PR #2231 `_] @@ -15,11 +17,11 @@ New features * Let storage scheduling infer missing ``power-capacity`` from directional device capacities before falling back to site capacity, and default the missing opposite capacity to zero when only a non-zero ``consumption-capacity`` or ``production-capacity`` is configured [see `PR #2222 `_] * CLI support for adding/editing account attributes [see `PR #2242 `_] - Infrastructure / Support ---------------------- * Upgraded dependencies [see `PR #1485 `_ and `PR #2215 `_] * Prepare the ``device_scheduler`` to deal with commitments per device group [see `PR #1934 `_] +* Support storing encrypted connection secrets on organisations and assets, including utility functions, encryption key configuration, CLI commands to set and delete secrets, and UI tables that show stored secret names and optional expiration times without exposing their values [see `PR #2236 `_] * Documentation section on the modelling choice for recording measurements, forecasts and schedules under one or multiple sensors [see `PR #2217 `_] Bugfixes diff --git a/documentation/cli/change_log.rst b/documentation/cli/change_log.rst index 5ee9129196..12eb9709a1 100644 --- a/documentation/cli/change_log.rst +++ b/documentation/cli/change_log.rst @@ -4,6 +4,12 @@ FlexMeasures CLI Changelog ********************** +since v1.0.0 | July XX, 2026 +================================= + +* Add ``flexmeasures edit secret`` to store an encrypted secret on an account or asset. +* Add ``flexmeasures delete secret`` to remove an encrypted secret from an account or asset. + since v0.33.0 | June 01, 2026 ================================= diff --git a/documentation/cli/commands.rst b/documentation/cli/commands.rst index 993a4906ec..1cd143e1c0 100644 --- a/documentation/cli/commands.rst +++ b/documentation/cli/commands.rst @@ -68,12 +68,12 @@ of which some are referred to in this documentation. ================================================= ======================================= ``flexmeasures edit attribute`` Edit (or add) an asset attribute or sensor attribute. +``flexmeasures edit secret`` Edit (or add) an encrypted secret on an account or asset. ``flexmeasures edit resample-data`` Assign a new event resolution to an existing sensor and resample its data accordingly. ``flexmeasures edit transfer-parenthood`` (Re)assign parent assets. ``flexmeasures edit transfer-ownership`` Transfer the ownership of an asset and its children to a different account. ================================================= ======================================= - ``delete`` - Delete data -------------- @@ -83,6 +83,7 @@ of which some are referred to in this documentation. ``flexmeasures delete account`` Delete a tenant account & also their users (with assets and power measurements). ``flexmeasures delete user`` Delete a user & also their assets and power measurements. ``flexmeasures delete asset`` Delete an asset & also its sensors and data. +``flexmeasures delete secret`` Delete an encrypted secret from an account or asset. ``flexmeasures delete sensor`` Delete a sensor and all beliefs about it. ``flexmeasures delete beliefs`` Delete time series data (beliefs). ``flexmeasures delete measurements`` Delete measurements (with horizon <= 0). diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 1b823aa585..3b27aab60c 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -508,6 +508,25 @@ Extra password salt (a.k.a. pepper) Default: ``None`` (falls back to ``SECRET_KEY``\ ) +.. _flexmeasures_secrets_encryption_keys: + +FLEXMEASURES_SECRETS_ENCRYPTION_KEYS +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Keyring (dictionary with key IDs mapped to key material), used to encrypt and decrypt +connection secrets stored for accounts and assets. This follows the same shape +as ``SECURITY_TOTP_SECRETS``, for example +``{"1": "old-secret", "2": "current-secret"}``. + +Newly encrypted values use the highest numeric key ID in this dictionary, or +the last key ID in lexical order if the IDs are not numeric. Keep previous keys +in this dictionary until all stored secrets have been re-encrypted with the +current key. + +This setting must be configured before connection secrets can be stored or decrypted. + +Default: ``None`` + SECURITY_TOKEN_AUTHENTICATION_HEADER ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/documentation/dev/connection-secrets.rst b/documentation/dev/connection-secrets.rst new file mode 100644 index 0000000000..7a15f4edab --- /dev/null +++ b/documentation/dev/connection-secrets.rst @@ -0,0 +1,71 @@ +.. _connection_secrets_dev: + +Connection secrets +================== + +If you make a secure connection to an external platform, FlexMeasures can store credentials, +API keys and tokens in the ``secrets`` JSON field of the account or asset that owns the connection. +Each secret value is encrypted separately and stored together with metadata such as its encryption +key ID and timestamps. Developers normally do not need to read or modify this +JSON structure directly. + +For implementation examples, token lifecycle strategies and manually seeding a +credential through the CLI, see :ref:`storing_connection_secrets`. + +The encrypted values are protected by +``FLEXMEASURES_SECRETS_ENCRYPTION_KEYS``, see :ref:`flexmeasures_secrets_encryption_keys` + This setting accepts arbitrary non-empty strings, which FlexMeasures derives into Fernet-compatible keys. +Hosts must configure this keyring before secrets can be stored +- FlexMeasures will print a warning if it is not set and hints how to initialize it. + + +Recommended practices +--------------------- + +Write-only API and UI +^^^^^^^^^^^^^^^^^^^^^ + +Treat secrets as write-only in API and UI flows. Accept new or replacement +values, but never return plaintext secrets. Responses should contain only +redacted information such as whether a value is set and when it expires. + +For administrator-level maintenance, use ``flexmeasures edit secret`` to store +or replace one account or asset secret and ``flexmeasures delete secret`` to +remove one. Use ``--secret`` for dot-separated paths or ``--secret-path`` once +or twice. Prefer the edit command's ``--prompt`` option so secret values do not +enter shell history. + + +Use the secret utilities +^^^^^^^^^^^^^^^^^^^^^^^^ + +FlexMeasures provides a complete utility API in :mod:`flexmeasures.utils.secrets_utils`. + +Use :func:`flexmeasures.utils.secrets_utils.set_secret` for individual +credentials. For token refresh flows, prefer +:class:`flexmeasures.utils.secrets_utils.TokenRefreshResult` together with +:func:`flexmeasures.utils.secrets_utils.apply_token_refresh_result`. They safely +handle replacement tokens, rotated refresh tokens and providers that only +extend the expiry of an existing token. + +Use :func:`flexmeasures.utils.secrets_utils.get_secret_overview` to build safe +secret listings with their paths and optional expiration times, without +exposing encrypted values or unrelated metadata. +When ``expires_at`` is stored without a timezone offset, FlexMeasures treats it +as UTC when rendering the overview. + +Store secrets at one or two path levels only: a top-level namespace such as a +provider name, optionally followed by a secret name within that namespace. +String paths therefore look like ``provider`` or ``provider.refresh_token``. +If a secret name itself contains a dot, pass the path as a tuple or list of +parts, or use ``--secret-path`` in the CLI. + + +Refresh early in multi-worker deployments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Store reusable access tokens and their expiry metadata in ``secrets`` so all +workers share the same token state. Refresh with time to spare before expiry to +allow for clock differences, retries and concurrent workers. In high-traffic +integrations, use database locking so only one worker performs a refresh. See +the refresh-leeway example in :ref:`initiating_connection_tokens`. diff --git a/documentation/index.rst b/documentation/index.rst index a771ccb589..869aa57d34 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -244,6 +244,7 @@ In :ref:`getting_started`, we have some helpful tips how to dive into this docum plugin/introduction plugin/showcase plugin/customisation + plugin/storing-platform-secrets .. toctree:: @@ -258,6 +259,7 @@ In :ref:`getting_started`, we have some helpful tips how to dive into this docum dev/auth dev/dependency-management dev/api + dev/connection-secrets dev/automated-deploy-via-GHActions .. autosummary:: @@ -288,4 +290,3 @@ In :ref:`getting_started`, we have some helpful tips how to dive into this docum .. * :ref:`genindex` .. * :ref:`modindex` .. * :ref:`search` - diff --git a/documentation/plugin/introduction.rst b/documentation/plugin/introduction.rst index 8a0eff8513..c453746981 100644 --- a/documentation/plugin/introduction.rst +++ b/documentation/plugin/introduction.rst @@ -32,4 +32,6 @@ To hit the ground running with that approach, we provide a `CookieCutter templat It also includes a few Blueprint examples and best practices. -Continue reading the :ref:`plugin_showcase` or possibilities to do :ref:`plugin_customization`. \ No newline at end of file +Continue reading the :ref:`plugin_showcase`. Continue to see more possibilities to do :ref:`plugin_customization`. +And if your plugin connects to a 3rd-party platform, read about support for :ref:`storing_connection_secrets`. + diff --git a/documentation/plugin/storing-platform-secrets.rst b/documentation/plugin/storing-platform-secrets.rst new file mode 100644 index 0000000000..e6a03c7eee --- /dev/null +++ b/documentation/plugin/storing-platform-secrets.rst @@ -0,0 +1,168 @@ +.. _storing_connection_secrets: + +Storing connection secrets +========================== + +Plugins that connect FlexMeasures accounts or assets to external platforms often +need to store credentials, refresh tokens, access tokens or connection-specific +passwords. Store such values in the ``secrets`` JSON field of the relevant +account or asset, rather than in ``attributes`` or plugin configuration files. + +Secrets live in a dictionary and support one or two path levels. For example: + +.. code-block:: json + + { + "3rdparty-platform": { + "refresh_token": "encrypted-value", + "access_token": "encrypted-value" + } + } + +(where "encrypted-value" is not the actual string you pass in, FlexMeasures handles the encryption seamlessly) + +The ``secrets`` field is intended to be write-only from API and UI flows: users +can provide or replace secret values, but normal responses should return only +redacted metadata such as whether a value is set and when it expires. Trusted +server-side plugin code can decrypt and use the value when it performs work for +the account or asset. + +Use ``flexmeasures.utils.secrets_utils`` for secret handling: + +.. code-block:: python + + from flexmeasures.utils.secrets_utils import ( + SecretsEncryptor, + get_secret, + redact_secrets, + set_secret, + ) + + encryptor = SecretsEncryptor.from_current_app() + + my_account.secrets = set_secret( + my_account.secrets, + "3rdparty-platform.refresh_token", + refresh_token, + encryptor=encryptor, + metadata={"expires_at": refresh_token_expires_at.isoformat()}, + ) + + refresh_token = get_secret( + my_account.secrets, + "3rdparty-platform.refresh_token", + encryptor=encryptor, + ) + + response_payload = redact_secrets(my_account.secrets) + +The metadata string ``expires_at`` (see example above) is useful to add, as it will be used by the token handling and also shown in the UI. +FlexMeasures will also handle ``created-at`` and ``updated-at`` metadata automatically. + +More details and best practices for storing connection secrets are in the :ref:`connection_secrets_dev` section. + + +Token lifecycle strategies +----------------------------- + +External platforms do not all require the same interactions to maintain a connection. +Initial login, token refresh and token expiry can all work rather differently. +You could say that they implement different "token lifecycle strategies". + +That's why our advice is to keep the provider-specific HTTP calls in your plugin, +and use FlexMeasures utilities only for encryption, redaction and updating stored token state. + +The following token lifecycle strategies are supported by +``TokenRefreshResult`` and ``apply_token_refresh_result``: + +* A refresh operation returns a new access token: set + ``TokenRefreshResult.access_token`` and ``access_token_expires_at``. +* A refresh operation rotates the refresh token: set + ``TokenRefreshResult.refresh_token`` and ``refresh_token_expires_at``. +* A refresh operation extends an existing access token without returning a new token: + leave ``access_token`` as ``None`` and set ``access_token_expires_at``. +* Access tokens are minted separately from refresh-token rotation: call + ``apply_token_refresh_result`` once for the refreshed long-lived credential + and again when a short-lived access token is minted. + +Your task is to translate the HTTP response from the platform provider into a +``TokenRefreshResult`` and let FlexMeasures update the encrypted JSON state. +Let's look at an example: + +.. code-block:: python + + from flexmeasures.utils.secrets_utils import ( + SecretsEncryptor, + TokenRefreshResult, + apply_token_refresh_result, + get_secret, + ) + + encryptor = SecretsEncryptor.from_current_app() + + refresh_token = get_secret( + my_account.secrets, + "3rdparty-platform.refresh_token", + encryptor=encryptor, + ) + + # this function would be written by you + token_response = refresh_with_external_platform(refresh_token) + + # here you translate the response - consult the platform docs + my_account.secrets = apply_token_refresh_result( + my_account.secrets, + "3rdparty-platform", + TokenRefreshResult( + access_token=token_response.get("access_token"), + refresh_token=token_response.get("refresh_token"), + access_token_expires_at=token_response.get("access_token_expires_at"), + refresh_token_expires_at=token_response.get("refresh_token_expires_at"), + token_type=token_response.get("token_type"), + metadata={"scope": token_response.get("scope")}, + ), + encryptor=encryptor, + ) + +This varies by platform - for instance, if a platform only extends the lifetime +of the existing access token (instead of returning a new one), keep +``access_token`` set to ``None`` and provide the new ``access_token_expires_at``. +The existing encrypted token is kept, and only its metadata is updated. + +Your plugin should also decide when to request a refreshed access token, e.g.: + +.. code-block:: python + + 3RDPARTY_PLATFORM_TOKEN_LEEWAY = timedelta(seconds=120) + # you might get this info with `secret_utils.get_secret()` + if current_access_token_expires_at > now + 3RDPARTY_PLATFORM_TOKEN_LEEWAY: + return access_token + response = send_request_to_3rdparty_platform() + access_token = response.text + refresh_account.secrets = apply_token_refresh_result(...) + + + +.. _initiating_connection_tokens: + +Initiating tokens (before app startup) +----------------------------------------- + +Most providers will require the true credentials only in the first interaction: +For example: username & password to get the access & refresh token. +From then on, the refresh token helps to get by (as long as it does not expire). + +In your plugin, you can write a CLI command to perform this login, get your first +refresh token and save it as a secret (see `set_secret()` and `apply_token_refresh_result()` +in utils/secrets_utils.py). +The token lifecycle strategy (see above) will depend on the platform you connect to. + +Also advisable: if no current token is in the database, let your plugin code fail +explicitly and advise the user to call your login CLI command. + + +Alternatively, you can manually store a known credential: Use ``flexmeasures edit secret`` with +an account or asset ID, plus either ``--secret`` or one or two ``--secret-path`` +options, and either ``--value`` or ``--prompt`` (to paste the secret instead of typing it). +Use ``--metadata`` for non-secret JSON metadata such as expiry +timestamps. diff --git a/documentation/views/asset-data.rst b/documentation/views/asset-data.rst index 62e29d39a5..74766fd947 100644 --- a/documentation/views/asset-data.rst +++ b/documentation/views/asset-data.rst @@ -162,6 +162,8 @@ Properties page --------------- The properties page allows you to view and edit the properties of the asset. +It lists stored secret names without exposing their values and shows a +human-readable expiration time when one is available. You can also delete the asset by clicking on the "Delete this asset" button. @@ -204,4 +206,3 @@ This is how the audit log looks for the history of actions taken on an asset: .. :scale: 40% | - diff --git a/flexmeasures/app.py b/flexmeasures/app.py index 2a03b6854e..ee9fcab76a 100644 --- a/flexmeasures/app.py +++ b/flexmeasures/app.py @@ -46,11 +46,10 @@ def create( # noqa C901 get_flexmeasures_env, ) from flexmeasures.utils.app_utils import ( - set_secret_key, - set_totp_secrets, init_sentry, ) from flexmeasures.utils.error_utils import add_basic_error_handlers + from flexmeasures.utils.secrets_utils import set_secret_key, set_totp_secrets configure_logging() # do this first, see https://flask.palletsprojects.com/en/2.0.x/logging cfg_location = find_flexmeasures_cfg() # Find flexmeasures.cfg location diff --git a/flexmeasures/cli/data_delete.py b/flexmeasures/cli/data_delete.py index ee7586febd..6b1fbb7431 100644 --- a/flexmeasures/cli/data_delete.py +++ b/flexmeasures/cli/data_delete.py @@ -35,6 +35,37 @@ DeprecatedOptionsCommand, ) from flexmeasures.utils.flexmeasures_inflection import join_words_into_a_list +from flexmeasures.utils.secrets_utils import delete_secret, get_secret_paths + + +def _resolve_secret_path( + secret: str | None, secret_path_parts: tuple[str, ...] +) -> str | tuple[str, ...]: + """Normalize CLI secret path arguments to a utility-friendly path.""" + if secret is not None and secret_path_parts: + raise ValueError("Pass either --secret or --secret-path, not both.") + if secret is None and not secret_path_parts: + raise ValueError("Pass either --secret or --secret-path.") + if len(secret_path_parts) > 2: + raise ValueError("Pass --secret-path at most twice.") + if secret_path_parts: + return secret_path_parts + assert secret is not None + return secret + + +def _count_affected_secrets( + secrets: dict | None, secret_path: str | tuple[str, ...] +) -> int: + """Count stored secrets that would be removed by deleting ``secret_path``.""" + if isinstance(secret_path, str): + prefix = secret_path + else: + prefix = ".".join(secret_path) + return sum( + path == prefix or path.startswith(f"{prefix}.") + for path in get_secret_paths(secrets) + ) @click.group("delete") @@ -42,6 +73,93 @@ def fm_delete_data(): """FlexMeasures: Delete data.""" +@fm_delete_data.command("secret") +@with_appcontext +@click.option( + "--account", + "account", + required=False, + type=AccountIdField(), + help="Delete a secret from this account. Follow up with the account's ID.", +) +@click.option( + "--asset", + "asset", + required=False, + type=AssetIdField(), + help="Delete a secret from this asset. Follow up with the asset's ID.", +) +@click.option( + "--secret", + "secret", + required=False, + help="Delete the secret with this name. Can also be a dot-separated path (maximally one dot).", +) +@click.option( + "--secret-path", + "secret_path_parts", + required=False, + multiple=True, + help="Delete the secret with this path part. Pass once or twice. Use this instead of --secret if your secret name contains a dot.", +) +@click.option( + "--force/--no-force", + default=False, + help="Delete without asking for confirmation.", +) +def delete_stored_secret( + account: Account | None, + asset: GenericAsset | None, + secret: str | None, + secret_path_parts: tuple[str, ...], + force: bool, +) -> None: + """Delete one encrypted secret from an account or asset. + + The command accepts exactly one account or asset and asks for confirmation + unless ``--force`` is used. + + \b + Examples: + flexmeasures delete secret --account 1 --secret platform.access_token + flexmeasures delete secret --asset 2 --secret-path platform --secret-path token.v2 --force + flexmeasures delete secret --asset 2 --secret platform.password --force + """ + if (account is None) == (asset is None): + raise ValueError("Pass exactly one of --account or --asset.") + resolved_secret_path = _resolve_secret_path(secret, secret_path_parts) + display_secret_path = secret or ".".join(secret_path_parts) + + if account is not None: + target: Account | GenericAsset = account + target_type = "account" + else: + assert asset is not None + target = asset + target_type = "asset" + if not force: + affected_count = _count_affected_secrets(target.secrets, resolved_secret_path) + affected_clause = ( + f" This affects {affected_count} stored secrets." + if affected_count > 1 + else "" + ) + click.confirm( + f"Delete secret '{display_secret_path}' from {target_type} {target.id}?{affected_clause}", + abort=True, + ) + + try: + target.secrets = delete_secret(target.secrets, resolved_secret_path) + except KeyError: + abort(f"Secret path '{display_secret_path}' does not exist.") + db.session.add(target) + db.session.commit() + done( + f"Secret '{display_secret_path}' has been deleted from {target_type} {target.id}." + ) + + @fm_delete_data.command("account-role") @with_appcontext @click.option("--name", required=True) diff --git a/flexmeasures/cli/data_edit.py b/flexmeasures/cli/data_edit.py index 00a968797b..8505013cfa 100644 --- a/flexmeasures/cli/data_edit.py +++ b/flexmeasures/cli/data_edit.py @@ -1,10 +1,8 @@ -""" -CLI commands for editing data -""" +"""CLI commands for editing data.""" from __future__ import annotations -from datetime import timedelta +from datetime import datetime, timedelta import click import pandas as pd @@ -31,6 +29,23 @@ abort, ) from flexmeasures.utils.flexmeasures_inflection import pluralize +from flexmeasures.utils.secrets_utils import store_account_secret, store_asset_secret + + +def _resolve_secret_path( + secret: str | None, secret_path_parts: tuple[str, ...] +) -> str | tuple[str, ...]: + """Normalize CLI secret path arguments to a utility-friendly path.""" + if secret is not None and secret_path_parts: + raise ValueError("Pass either --secret or --secret-path, not both.") + if secret is None and not secret_path_parts: + raise ValueError("Pass either --secret or --secret-path.") + if len(secret_path_parts) > 2: + raise ValueError("Pass --secret-path at most twice.") + if secret_path_parts: + return secret_path_parts + assert secret is not None + return secret @click.group("edit") @@ -185,6 +200,100 @@ def edit_attribute( click.secho("Successfully edited/added attribute.", **MsgStyle.SUCCESS) +@fm_edit_data.command("secret") +@with_appcontext +@click.option( + "--account", + "account", + required=False, + type=AccountIdField(), + help="Add/edit secret on this account. Follow up with the account's ID.", +) +@click.option( + "--asset", + "asset", + required=False, + type=AssetIdField(), + help="Add/edit secret on this asset. Follow up with the asset's ID.", +) +@click.option( + "--secret", + "secret", + required=False, + help="Add/edit this secret. Follow up with a secret name. Can also be a dot-separated path (maximally one dot), so the secret can be stored under a platform name (part before the dot).", +) +@click.option( + "--secret-path", + "secret_path_parts", + required=False, + multiple=True, + help="Add/edit secret with this path part. Pass once or twice. Use this instead of --secret if your secret name contains a dot.", +) +@click.option( + "--value", + "secret_value", + required=False, + type=str, + help="Set the secret to this string value.", +) +@click.option( + "--prompt", + "prompt_for_secret", + required=False, + is_flag=True, + default=False, + help="Prompt for the secret value without echoing it.", +) +@click.option( + "--metadata", + "metadata_json", + required=False, + type=str, + help="Non-secret metadata to store with the encrypted value, as a JSON object.", +) +def edit_secret( + account: Account | None, + asset: GenericAsset | None, + secret: str | None, + secret_path_parts: tuple[str, ...], + prompt_for_secret: bool, + secret_value: str | None = None, + metadata_json: str | None = None, +): + """Edit (or add) an encrypted account or asset secret. + + The command accepts exactly one account or asset. Prefer ``--prompt`` over + ``--value`` to avoid putting sensitive values in shell history. + + \b + Examples: + flexmeasures edit secret --account 1 --secret platform.refresh_token --prompt + flexmeasures edit secret --asset 2 --secret-path platform --secret-path token.v2 --prompt + flexmeasures edit secret --asset 2 --secret platform.password --value secret --metadata '{"expires_at": "2026-06-24T02:00:00"}' + """ + if (account is None) == (asset is None): + raise ValueError("Pass exactly one of --account or --asset.") + if (secret_value is None) == (not prompt_for_secret): + raise ValueError("Pass exactly one of --value or --prompt.") + resolved_secret_path = _resolve_secret_path(secret, secret_path_parts) + if prompt_for_secret: + secret_value = click.prompt("Secret value", hide_input=True) + assert secret_value is not None + + metadata = parse_secret_metadata(metadata_json) + + if account is not None: + store_account_secret( + account, resolved_secret_path, secret_value, metadata=metadata + ) + db.session.add(account) + if asset is not None: + store_asset_secret(asset, resolved_secret_path, secret_value, metadata=metadata) + db.session.add(asset) + db.session.commit() + click.secho("Successfully edited/added secret.", **MsgStyle.SUCCESS) + + @fm_edit_data.command("resample-data", cls=DeprecatedOptionsCommand) @with_appcontext @click.option( @@ -487,6 +596,31 @@ def parse_attribute_value( # noqa: C901 return attribute_str_value +def parse_secret_metadata(metadata_json: str | None = None) -> dict | None: + """Parse secret metadata from a JSON object.""" + if metadata_json is None: + return None + try: + metadata = json.loads(metadata_json) + except json.decoder.JSONDecodeError as jde: + raise ValueError(f"Error parsing secret metadata: {jde}") + if not isinstance(metadata, dict): + raise ValueError("Secret metadata must be a JSON object.") + expires_at = metadata.get("expires_at") + if expires_at is not None: + if not isinstance(expires_at, str): + raise ValueError("Secret metadata field 'expires_at' must be a string.") + try: + datetime.fromisoformat( + f"{expires_at[:-1]}+00:00" if expires_at.endswith("Z") else expires_at + ) + except ValueError as exc: + raise ValueError( + "Secret metadata field 'expires_at' must be a valid ISO datetime." + ) from exc + return metadata + + def single_true(iterable) -> bool: i = iter(iterable) return any(i) and not any(i) diff --git a/flexmeasures/cli/tests/test_data_delete.py b/flexmeasures/cli/tests/test_data_delete.py index 7fd9a2b558..a843e9cea3 100644 --- a/flexmeasures/cli/tests/test_data_delete.py +++ b/flexmeasures/cli/tests/test_data_delete.py @@ -1,9 +1,216 @@ +import pytest from sqlalchemy import select, func from flexmeasures.cli.tests.utils import check_command_ran_without_error, to_flags from flexmeasures.data.models.audit_log import AuditLog from flexmeasures.data.models.user import Account, User from flexmeasures.data.services.users import find_user_by_email +from flexmeasures.utils.secrets_utils import ( + store_account_secret, + store_asset_secret, +) + + +def test_delete_account_secret(app, fresh_db, setup_accounts_fresh_db): + from flexmeasures.cli.data_delete import delete_stored_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + account = setup_accounts_fresh_db["Prosumer"] + store_account_secret(account, "platform.access_token", "access-token") + store_account_secret(account, "platform.refresh_token", "refresh-token") + fresh_db.session.commit() + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = None + + runner = app.test_cli_runner() + result = runner.invoke( + delete_stored_secret, + to_flags( + { + "account": account.id, + "secret": "platform.access_token", + } + ) + + ["--force"], + ) + + check_command_ran_without_error(result) + assert "access-token" not in result.output + fresh_db.session.refresh(account) + assert "access_token" not in account.secrets["platform"] + assert "refresh_token" in account.secrets["platform"] + + +def test_delete_asset_secret_prunes_empty_parents( + app, fresh_db, setup_generic_assets_fresh_db +): + from flexmeasures.cli.data_delete import delete_stored_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + asset = setup_generic_assets_fresh_db["test_battery"] + store_asset_secret(asset, "platform.password", "password-value") + fresh_db.session.commit() + + runner = app.test_cli_runner() + result = runner.invoke( + delete_stored_secret, + to_flags( + { + "asset": asset.id, + "secret": "platform.password", + } + ) + + ["--force"], + ) + + check_command_ran_without_error(result) + assert "password-value" not in result.output + fresh_db.session.refresh(asset) + assert asset.secrets == {} + + +def test_delete_secret_confirmation_mentions_multiple_affected_secrets( + app, fresh_db, setup_accounts_fresh_db +): + from flexmeasures.cli.data_delete import delete_stored_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + account = setup_accounts_fresh_db["Prosumer"] + store_account_secret(account, "platform.access_token", "access-token") + store_account_secret(account, "platform.refresh_token", "refresh-token") + fresh_db.session.commit() + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = None + + runner = app.test_cli_runner() + result = runner.invoke( + delete_stored_secret, + [ + "--account", + str(account.id), + "--secret", + "platform", + ], + input="n\n", + ) + + assert result.exit_code == 1 + assert "Delete secret 'platform' from account" in result.output + assert "This affects 2 stored secrets." in result.output + fresh_db.session.refresh(account) + assert "platform" in account.secrets + + +def test_delete_secret_accepts_secret_path_with_dot_in_leaf( + app, fresh_db, setup_generic_assets_fresh_db +): + from flexmeasures.cli.data_delete import delete_stored_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + asset = setup_generic_assets_fresh_db["test_battery"] + store_asset_secret(asset, ("platform", "token.v2"), "password-value") + fresh_db.session.commit() + + runner = app.test_cli_runner() + result = runner.invoke( + delete_stored_secret, + [ + "--asset", + str(asset.id), + "--secret-path", + "platform", + "--secret-path", + "token.v2", + "--force", + ], + ) + + check_command_ran_without_error(result) + fresh_db.session.refresh(asset) + assert asset.secrets == {} + + +def test_delete_secret_rejects_account_and_asset( + app, fresh_db, setup_accounts_fresh_db, setup_generic_assets_fresh_db +): + from flexmeasures.cli.data_delete import delete_stored_secret + + fresh_db.session.flush() + account = setup_accounts_fresh_db["Prosumer"] + asset = setup_generic_assets_fresh_db["test_battery"] + runner = app.test_cli_runner() + + with pytest.raises(ValueError, match="Pass exactly one of --account or --asset."): + runner.invoke( + delete_stored_secret, + to_flags( + { + "account": account.id, + "asset": asset.id, + "secret": "platform.password", + } + ) + + ["--force"], + ) + + +def test_delete_secret_rejects_secret_and_secret_path_together( + app, fresh_db, setup_accounts_fresh_db +): + from flexmeasures.cli.data_delete import delete_stored_secret + + account = setup_accounts_fresh_db["Prosumer"] + runner = app.test_cli_runner() + + with pytest.raises( + ValueError, match="Pass either --secret or --secret-path, not both." + ): + runner.invoke( + delete_stored_secret, + [ + "--account", + str(account.id), + "--secret", + "platform.password", + "--secret-path", + "platform", + "--force", + ], + ) + + +def test_delete_secret_rejects_more_than_two_secret_path_parts( + app, fresh_db, setup_accounts_fresh_db +): + from flexmeasures.cli.data_delete import delete_stored_secret + + account = setup_accounts_fresh_db["Prosumer"] + runner = app.test_cli_runner() + + with pytest.raises(ValueError, match="Pass --secret-path at most twice."): + runner.invoke( + delete_stored_secret, + [ + "--account", + str(account.id), + "--secret-path", + "platform", + "--secret-path", + "nested", + "--secret-path", + "token", + "--force", + ], + ) + + +def test_delete_secret_help_includes_examples(app): + from flexmeasures.cli.data_delete import delete_stored_secret + + result = app.test_cli_runner().invoke(delete_stored_secret, ["--help"]) + + check_command_ran_without_error(result) + assert "Examples:" in result.output + assert "flexmeasures delete secret --account" in result.output + assert "--secret-path platform --secret-path token.v2 --force" in result.output def test_delete_account( diff --git a/flexmeasures/cli/tests/test_data_edit.py b/flexmeasures/cli/tests/test_data_edit.py index 9a5dbee6ad..11058164ec 100644 --- a/flexmeasures/cli/tests/test_data_edit.py +++ b/flexmeasures/cli/tests/test_data_edit.py @@ -9,6 +9,7 @@ from flexmeasures.data.models.time_series import TimedBelief from flexmeasures.cli.tests.utils import check_attribute_is_stored, get_click_commands from flexmeasures.tests.utils import get_test_sensor +from flexmeasures.utils.secrets_utils import get_secret def test_add_one_sensor_attribute(app, db, setup_markets): @@ -102,6 +103,237 @@ def test_update_one_account_attribute(app, db, setup_generic_assets): check_attribute_is_stored(account, "some-attribute", "some-new-value") +def test_edit_account_secret(app, db, setup_accounts): + from flexmeasures.cli.data_edit import edit_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + account = setup_accounts["Prosumer"] + cli_input = { + "account": account.id, + "secret": "platform.refresh_token", + "value": "refresh-token-value", + "metadata": '{"expires_at": "2026-06-11T12:00:00+00:00"}', + } + + runner = app.test_cli_runner() + result = runner.invoke(edit_secret, to_flags(cli_input)) + check_command_ran_without_error(result) + assert "Success" in result.output, result.exception + assert "refresh-token-value" not in result.output + + db.session.refresh(account) + envelope = account.secrets["platform"]["refresh_token"] + assert envelope["ciphertext"] != "refresh-token-value" + assert envelope["expires_at"] == "2026-06-11T12:00:00+00:00" + assert get_secret(account.secrets, "platform.refresh_token") == ( + "refresh-token-value" + ) + + +def test_edit_secret_accepts_naive_iso_expires_at_metadata(app, db, setup_accounts): + from flexmeasures.cli.data_edit import edit_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + account = setup_accounts["Prosumer"] + runner = app.test_cli_runner() + + result = runner.invoke( + edit_secret, + [ + "--account", + str(account.id), + "--secret", + "platform.refresh_token", + "--value", + "refresh-token-value", + "--metadata", + '{"expires_at": "2026-06-11T12:00:00"}', + ], + ) + + check_command_ran_without_error(result) + db.session.refresh(account) + assert account.secrets["platform"]["refresh_token"]["expires_at"] == ( + "2026-06-11T12:00:00" + ) + + +def test_edit_asset_secret(app, db, setup_generic_assets): + from flexmeasures.cli.data_edit import edit_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + asset = setup_generic_assets["test_battery"] + db.session.flush() + cli_input = { + "asset": asset.id, + "secret": "platform.password", + "value": "password-value", + } + + runner = app.test_cli_runner() + result = runner.invoke(edit_secret, to_flags(cli_input)) + check_command_ran_without_error(result) + assert "Success" in result.output, result.exception + assert "password-value" not in result.output + + db.session.refresh(asset) + envelope = asset.secrets["platform"]["password"] + assert envelope["ciphertext"] != "password-value" + assert get_secret(asset.secrets, "platform.password") == "password-value" + + +def test_edit_secret_accepts_secret_path_with_dot_in_leaf( + app, db, setup_generic_assets +): + from flexmeasures.cli.data_edit import edit_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + asset = setup_generic_assets["test_battery"] + db.session.flush() + + runner = app.test_cli_runner() + result = runner.invoke( + edit_secret, + [ + "--asset", + str(asset.id), + "--secret-path", + "platform", + "--secret-path", + "token.v2", + "--value", + "password-value", + ], + ) + + check_command_ran_without_error(result) + assert "password-value" not in result.output + db.session.refresh(asset) + assert get_secret(asset.secrets, ("platform", "token.v2")) == "password-value" + + +def test_edit_secret_rejects_account_and_asset( + app, db, setup_accounts, setup_generic_assets +): + from flexmeasures.cli.data_edit import edit_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + account = setup_accounts["Prosumer"] + asset = setup_generic_assets["test_battery"] + db.session.flush() + cli_input = { + "account": account.id, + "asset": asset.id, + "secret": "platform.password", + "value": "password-value", + } + + runner = app.test_cli_runner() + with pytest.raises(ValueError, match="Pass exactly one of --account or --asset."): + runner.invoke(edit_secret, to_flags(cli_input)) + + +def test_edit_secret_rejects_invalid_iso_expires_at_metadata(app, db, setup_accounts): + from flexmeasures.cli.data_edit import edit_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + account = setup_accounts["Prosumer"] + runner = app.test_cli_runner() + + with pytest.raises( + ValueError, + match="Secret metadata field 'expires_at' must be a valid ISO datetime.", + ): + runner.invoke( + edit_secret, + [ + "--account", + str(account.id), + "--secret", + "platform.refresh_token", + "--value", + "refresh-token-value", + "--metadata", + '{"expires_at": "not-a-datetime"}', + ], + ) + + +def test_edit_secret_rejects_secret_and_secret_path_together(app, db, setup_accounts): + from flexmeasures.cli.data_edit import edit_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + account = setup_accounts["Prosumer"] + runner = app.test_cli_runner() + + with pytest.raises( + ValueError, match="Pass either --secret or --secret-path, not both." + ): + runner.invoke( + edit_secret, + [ + "--account", + str(account.id), + "--secret", + "platform.password", + "--secret-path", + "platform", + "--value", + "password-value", + ], + ) + + +def test_edit_secret_rejects_more_than_two_secret_path_parts(app, db, setup_accounts): + from flexmeasures.cli.data_edit import edit_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + account = setup_accounts["Prosumer"] + runner = app.test_cli_runner() + + with pytest.raises(ValueError, match="Pass --secret-path at most twice."): + runner.invoke( + edit_secret, + [ + "--account", + str(account.id), + "--secret-path", + "platform", + "--secret-path", + "nested", + "--secret-path", + "token", + "--value", + "password-value", + ], + ) + + +def test_edit_secret_rejects_missing_target(app): + from flexmeasures.cli.data_edit import edit_secret + + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = {"1": "test-master-key"} + cli_input = { + "secret": "platform.password", + "value": "password-value", + } + + runner = app.test_cli_runner() + with pytest.raises(ValueError, match="Pass exactly one of --account or --asset."): + runner.invoke(edit_secret, to_flags(cli_input)) + + +def test_edit_secret_help_includes_examples(app): + from flexmeasures.cli.data_edit import edit_secret + + result = app.test_cli_runner().invoke(edit_secret, ["--help"]) + + check_command_ran_without_error(result) + assert "Examples:" in result.output + assert "flexmeasures edit secret --account" in result.output + assert "--secret-path platform --secret-path token.v2" in result.output + + @pytest.mark.parametrize( "event_starts_after, event_ends_before", ( diff --git a/flexmeasures/data/migrations/versions/55d8936a55f9_merge_two_revision_heads.py b/flexmeasures/data/migrations/versions/55d8936a55f9_merge_two_revision_heads.py new file mode 100644 index 0000000000..3f45e3901e --- /dev/null +++ b/flexmeasures/data/migrations/versions/55d8936a55f9_merge_two_revision_heads.py @@ -0,0 +1,21 @@ +"""merge two revision heads + +Revision ID: 55d8936a55f9 +Revises: c7b7f9019d4b, c7d8e9f0a1b2 +Create Date: 2026-06-18 16:14:20.259079 + +""" + +# revision identifiers, used by Alembic. +revision = "55d8936a55f9" +down_revision = ("c7b7f9019d4b", "c7d8e9f0a1b2") +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/flexmeasures/data/migrations/versions/c7b7f9019d4b_allow_duplicate_root_asset_names.py b/flexmeasures/data/migrations/versions/c7b7f9019d4b_allow_duplicate_root_asset_names.py index aede6c26a6..b7966da005 100644 --- a/flexmeasures/data/migrations/versions/c7b7f9019d4b_allow_duplicate_root_asset_names.py +++ b/flexmeasures/data/migrations/versions/c7b7f9019d4b_allow_duplicate_root_asset_names.py @@ -19,7 +19,15 @@ def upgrade(): op.drop_constraint( - "generic_asset_name_parent_asset_id_key", "generic_asset", type_="unique" + "generic_asset_name_parent_asset_id_key", + "generic_asset", + type_="unique", + if_exists=True, + ) + op.drop_index( + "generic_asset_name_parent_asset_id_key", + table_name="generic_asset", + if_exists=True, ) op.create_index( "generic_asset_name_parent_asset_id_key", @@ -48,9 +56,21 @@ def upgrade(): def downgrade(): - op.drop_index("generic_asset_public_root_name_key", table_name="generic_asset") - op.drop_index("generic_asset_root_account_id_name_key", table_name="generic_asset") - op.drop_index("generic_asset_name_parent_asset_id_key", table_name="generic_asset") + op.drop_index( + "generic_asset_public_root_name_key", + table_name="generic_asset", + if_exists=True, + ) + op.drop_index( + "generic_asset_root_account_id_name_key", + table_name="generic_asset", + if_exists=True, + ) + op.drop_index( + "generic_asset_name_parent_asset_id_key", + table_name="generic_asset", + if_exists=True, + ) op.create_unique_constraint( "generic_asset_name_parent_asset_id_key", "generic_asset", diff --git a/flexmeasures/data/migrations/versions/c7d8e9f0a1b2_add_secrets_to_accounts_and_assets.py b/flexmeasures/data/migrations/versions/c7d8e9f0a1b2_add_secrets_to_accounts_and_assets.py new file mode 100644 index 0000000000..1dd59f1d7f --- /dev/null +++ b/flexmeasures/data/migrations/versions/c7d8e9f0a1b2_add_secrets_to_accounts_and_assets.py @@ -0,0 +1,38 @@ +"""Add encrypted secrets JSON fields to accounts and assets + +Revision ID: c7d8e9f0a1b2 +Revises: b2c3d4e5f6a7 +Create Date: 2026-06-11 + +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB + + +# revision identifiers, used by Alembic. +revision = "c7d8e9f0a1b2" +down_revision = "b2c3d4e5f6a7" +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table("account", schema=None) as batch_op: + batch_op.add_column( + sa.Column("secrets", JSONB(), nullable=False, server_default="{}") + ) + + with op.batch_alter_table("generic_asset", schema=None) as batch_op: + batch_op.add_column( + sa.Column("secrets", JSONB(), nullable=False, server_default="{}") + ) + + +def downgrade(): + with op.batch_alter_table("generic_asset", schema=None) as batch_op: + batch_op.drop_column("secrets") + + with op.batch_alter_table("account", schema=None) as batch_op: + batch_op.drop_column("secrets") diff --git a/flexmeasures/data/models/generic_assets.py b/flexmeasures/data/models/generic_assets.py index 2e580906ec..ad64be981d 100644 --- a/flexmeasures/data/models/generic_assets.py +++ b/flexmeasures/data/models/generic_assets.py @@ -244,6 +244,7 @@ class GenericAsset(db.Model, AuthModelMixin): latitude = db.Column(db.Float, nullable=True) longitude = db.Column(db.Float, nullable=True) attributes = db.Column(MutableDict.as_mutable(JSONB), nullable=False, default={}) + secrets = db.Column(MutableDict.as_mutable(JSONB), nullable=False, default={}) sensors_to_show = db.Column( MutableList.as_mutable(JSONB), nullable=False, default=[] ) @@ -303,6 +304,8 @@ def __init__(self, **kwargs): self.flex_model = {} if self.attributes is None: self.attributes = {} + if self.secrets is None: + self.secrets = {} # For backwards compatibility, when setting attributes that served as flex-model fields, # move them to the flex_context (and don't store them as attributes) diff --git a/flexmeasures/data/models/user.py b/flexmeasures/data/models/user.py index 2f0239dfec..82e045149f 100644 --- a/flexmeasures/data/models/user.py +++ b/flexmeasures/data/models/user.py @@ -69,6 +69,7 @@ class Account(db.Model, AuthModelMixin): secondary_color = Column(String(7), default=None) logo_url = Column(String(255), default=None) attributes = Column(MutableDict.as_mutable(JSONB), nullable=False, default={}) + secrets = Column(MutableDict.as_mutable(JSONB), nullable=False, default={}) annotations = db.relationship( "Annotation", secondary="annotations_accounts", diff --git a/flexmeasures/ui/templates/_macros.html b/flexmeasures/ui/templates/_macros.html index b69146880d..3d5d4a399d 100644 --- a/flexmeasures/ui/templates/_macros.html +++ b/flexmeasures/ui/templates/_macros.html @@ -204,6 +204,36 @@ {% endmacro %} +{% macro render_stored_secrets(stored_secrets) %} +{% if stored_secrets %} +{% set show_expiry = stored_secrets | selectattr("expires_at", "defined") | list %} + + + + + {% if show_expiry %} + + {% endif %} + + + + {% for secret in stored_secrets %} + + + {% if show_expiry %} + + {% endif %} + + {% endfor %} + +
Stored secretExpires
{{ secret.path }} + {% if secret.expires_at is defined %} + {{ secret.expires_at | naturalized_datetime }} + {% endif %} +
+{% endif %} +{% endmacro %} + {# Render a button that opens the JSON attributes editor modal. Call json_attributes_editor_modal() once on the same page to include the modal + JS. @@ -515,4 +545,3 @@
}()); {% endmacro %} - diff --git a/flexmeasures/ui/templates/accounts/account.html b/flexmeasures/ui/templates/accounts/account.html index d8dbc81c4e..82a2f99731 100644 --- a/flexmeasures/ui/templates/accounts/account.html +++ b/flexmeasures/ui/templates/accounts/account.html @@ -239,8 +239,9 @@

Account

- {% from "_macros.html" import render_attributes %} + {% from "_macros.html" import render_attributes, render_stored_secrets %} {{ render_attributes(account.attributes) }} + {{ render_stored_secrets(stored_secrets) }}
diff --git a/flexmeasures/ui/templates/assets/asset_properties.html b/flexmeasures/ui/templates/assets/asset_properties.html index 88842576b2..4564e7ef45 100644 --- a/flexmeasures/ui/templates/assets/asset_properties.html +++ b/flexmeasures/ui/templates/assets/asset_properties.html @@ -204,8 +204,9 @@

Asset Properties - {{ asset.name }}

- {% from "_macros.html" import render_attributes %} + {% from "_macros.html" import render_attributes, render_stored_secrets %} {{ render_attributes(asset.attributes) }} + {{ render_stored_secrets(stored_secrets) }}
@@ -1350,4 +1351,4 @@
Edit {{ asset.name }}'s flex-model
entity_name=asset.name ) }} {% endif %} - {% endblock %} \ No newline at end of file + {% endblock %} diff --git a/flexmeasures/ui/tests/test_json_attributes_editor.py b/flexmeasures/ui/tests/test_json_attributes_editor.py index 3c2f192dd9..67bbc47280 100644 --- a/flexmeasures/ui/tests/test_json_attributes_editor.py +++ b/flexmeasures/ui/tests/test_json_attributes_editor.py @@ -7,14 +7,85 @@ from __future__ import annotations +import re +from datetime import datetime + from flask import url_for from flask_login import current_user from flexmeasures.data.services.users import find_user_by_email +from flexmeasures.utils.time_utils import naturalized_datetime_str EDITOR_BUTTON_MARKER = b"Open editor" EDITOR_MODAL_MARKER = b"jsonAttributesModal" +SECRET_EXPIRES_AT = "2026-06-11T12:00:00+00:00" +SECRET_METADATA_MARKERS = ( + b"sensitive-ciphertext", + b"sensitive-key-id", + b"sensitive-created-at", + b"sensitive-updated-at", + b"sensitive-token-type", + b"sensitive-provider-metadata", +) + + +def find_stored_secrets_table(response_data: bytes) -> bytes: + table = re.search( + rb"]*>(?:(?!)[\s\S])*?Stored secret" + rb"(?:(?!)[\s\S])*", + response_data, + ) + assert table is not None + return table.group() + + +def assert_secret_metadata_is_safely_rendered(response_data: bytes) -> None: + stored_secrets_table = find_stored_secrets_table(response_data) + + def find_secret_row(path: bytes) -> re.Match[bytes] | None: + return re.search( + rb"]*>(?:(?!)[\s\S])*?" + + re.escape(path) + + rb"(?:(?!)[\s\S])*", + stored_secrets_table, + ) + + expiring_secret_row = find_secret_row(b"platform.access_token") + secret_without_expiry_row = find_secret_row(b"platform.refresh_token") + assert expiring_secret_row is not None + assert secret_without_expiry_row is not None + + human_expiry = naturalized_datetime_str( + datetime.fromisoformat(SECRET_EXPIRES_AT) + ).encode() + expiry_title = f'title="{SECRET_EXPIRES_AT}"'.encode() + assert b"Expires" in stored_secrets_table + assert human_expiry in expiring_secret_row.group() + assert human_expiry not in secret_without_expiry_row.group() + assert expiry_title in expiring_secret_row.group() + assert expiry_title not in secret_without_expiry_row.group() + for marker in SECRET_METADATA_MARKERS: + assert marker not in response_data + + +def secret_test_data() -> dict: + return { + "platform": { + "access_token": { + "ciphertext": "sensitive-ciphertext", + "key_id": "sensitive-key-id", + "created_at": "sensitive-created-at", + "updated_at": "sensitive-updated-at", + "expires_at": SECRET_EXPIRES_AT, + "token_type": "sensitive-token-type", + "provider_metadata": "sensitive-provider-metadata", + }, + "refresh_token": { + "ciphertext": "another-sensitive-ciphertext", + }, + } + } # --------------------------------------------------------------------------- @@ -75,6 +146,61 @@ def test_asset_properties_page_shows_attributes_editor( assert EDITOR_MODAL_MARKER in response.data +def test_asset_properties_page_shows_secret_paths_without_values( + db, client, setup_assets, as_admin +): + user = find_user_by_email("flexmeasures-admin@seita.nl") + asset = user.account.generic_assets[0] + original_secrets = asset.secrets + asset.secrets = secret_test_data() + db.session.commit() + + try: + response = client.get( + url_for("AssetCrudUI:properties", id=asset.id), + follow_redirects=True, + ) + assert response.status_code == 200 + assert b"Stored secret" in response.data + assert b"platform.access_token" in response.data + assert b"platform.refresh_token" in response.data + assert_secret_metadata_is_safely_rendered(response.data) + finally: + asset.secrets = original_secrets + db.session.commit() + + +def test_asset_properties_page_omits_expires_column_without_expiry_metadata( + db, client, setup_assets, as_admin +): + user = find_user_by_email("flexmeasures-admin@seita.nl") + asset = user.account.generic_assets[0] + original_secrets = asset.secrets + asset.secrets = { + "platform": { + "access_token": {"ciphertext": "sensitive-ciphertext"}, + "refresh_token": {"ciphertext": "another-sensitive-ciphertext"}, + } + } + db.session.commit() + + try: + response = client.get( + url_for("AssetCrudUI:properties", id=asset.id), + follow_redirects=True, + ) + assert response.status_code == 200 + stored_secrets_table = find_stored_secrets_table(response.data) + assert b"platform.access_token" in stored_secrets_table + assert b"platform.refresh_token" in stored_secrets_table + assert b"Expires" not in stored_secrets_table + assert b"sensitive-ciphertext" not in response.data + assert b"another-sensitive-ciphertext" not in response.data + finally: + asset.secrets = original_secrets + db.session.commit() + + # --------------------------------------------------------------------------- # Account page # --------------------------------------------------------------------------- @@ -89,3 +215,24 @@ def test_account_page_shows_attributes_editor(db, client, as_prosumer_user1): assert response.status_code == 200 assert EDITOR_BUTTON_MARKER in response.data assert EDITOR_MODAL_MARKER in response.data + + +def test_account_page_shows_secret_paths_without_values(db, client, as_prosumer_user1): + account = current_user.account + original_secrets = account.secrets + account.secrets = secret_test_data() + db.session.commit() + + try: + response = client.get( + url_for("AccountCrudUI:get", account_id=account.id), + follow_redirects=True, + ) + assert response.status_code == 200 + assert b"Stored secret" in response.data + assert b"platform.access_token" in response.data + assert b"platform.refresh_token" in response.data + assert_secret_metadata_is_safely_rendered(response.data) + finally: + account.secrets = original_secrets + db.session.commit() diff --git a/flexmeasures/ui/views/accounts.py b/flexmeasures/ui/views/accounts.py index 544011fd23..58fc5288b3 100644 --- a/flexmeasures/ui/views/accounts.py +++ b/flexmeasures/ui/views/accounts.py @@ -18,6 +18,7 @@ ATTRIBUTES_FIELD_LABEL, ATTRIBUTES_FIELD_DESCRIPTION, ) +from flexmeasures.utils.secrets_utils import get_secret_overview class AccountCrudUI(FlaskView): @@ -78,6 +79,7 @@ def get(self, account_id: str): asset_icon_map=ICON_MAPPING, attributes_label=ATTRIBUTES_FIELD_LABEL, attributes_description=ATTRIBUTES_FIELD_DESCRIPTION, + stored_secrets=get_secret_overview(account.secrets), breadcrumb_info=get_breadcrumb_info(account), ) diff --git a/flexmeasures/ui/views/assets/views.py b/flexmeasures/ui/views/assets/views.py index 384661fa6e..23bcc0733e 100644 --- a/flexmeasures/ui/views/assets/views.py +++ b/flexmeasures/ui/views/assets/views.py @@ -24,6 +24,7 @@ from flexmeasures.ui.utils.view_utils import ICON_MAPPING from flexmeasures.data.models.user import Account from flexmeasures.utils.time_utils import duration_isoformat +from flexmeasures.utils.secrets_utils import get_secret_overview from flexmeasures.ui.utils.view_utils import render_flexmeasures_template from flexmeasures.ui.views.assets.forms import NewAssetForm, AssetForm from flexmeasures.ui.views import ( @@ -475,4 +476,5 @@ def properties(self, id: str): current_page="Properties", attributes_label=ATTRIBUTES_FIELD_LABEL, attributes_description=ATTRIBUTES_FIELD_DESCRIPTION, + stored_secrets=get_secret_overview(asset.secrets), ) diff --git a/flexmeasures/utils/app_utils.py b/flexmeasures/utils/app_utils.py index 143421bd73..f890adf0ca 100644 --- a/flexmeasures/utils/app_utils.py +++ b/flexmeasures/utils/app_utils.py @@ -4,10 +4,6 @@ from __future__ import annotations -import os -import sys -import json - import click from flask import Flask, current_app, redirect from flask.cli import FlaskGroup, with_appcontext @@ -84,141 +80,6 @@ def init_sentry(app: Flask): sentry_sdk.set_tag("platform-name", app.config.get("FLEXMEASURES_PLATFORM_NAME")) -def set_secret_key(app, filename="secret_key"): - """Set the SECRET_KEY or exit. - - We first check if it is already in the config. - - Then we look for it in environment var SECRET_KEY. - - Finally, we look for `filename` in the app's instance directory. - - If nothing is found, we print instructions - to create the secret and then exit. - """ - secret_key = app.config.get("SECRET_KEY", None) - if secret_key is not None: - return - secret_key = os.environ.get("SECRET_KEY", None) - if secret_key is not None: - app.config["SECRET_KEY"] = secret_key - return - filename = os.path.join(app.instance_path, filename) - try: - app.config["SECRET_KEY"] = open(filename, "rb").read() - except IOError: - app.logger.error( - """ - Error: No secret key set. - - You can add the SECRET_KEY setting to your conf file (this example works only on Unix): - - echo "SECRET_KEY=\\"`python3 -c 'import secrets; print(secrets.token_hex(24))'`\\"" >> ~/.flexmeasures.cfg - - OR you can add an env var: - - export SECRET_KEY=xxxxxxxxxxxxxxx - (on windows, use "set" instead of "export") - - OR you can create a secret key file (this example works only on Unix): - - mkdir -p %s - head -c 24 /dev/urandom > %s - - You can also use Python to create a good secret: - - python -c "import secrets; print(secrets.token_urlsafe())" - - """ - % (os.path.dirname(filename), filename) - ) - - sys.exit(2) - - -def log_totp_secrets_error_and_exit(app, filename): - app.logger.error( - """ - ERROR: The file %s exists but does not contain a valid dictionary. - - The correct format is: - - {"1": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"} - - """ - % (filename) - ) - sys.exit(2) - - -def set_totp_secrets(app, filename="totp_secrets"): - """Set the SECURITY_TOTP_SECRETS or exit. - - We first check if it is already in the config. - - Then we look for it in environment var SECURITY_TOTP_SECRETS. - - Finally, we look for `filename` in the app's instance directory. - - If nothing is found, we print instructions - to create the secret and then exit. - """ - totp_secrets = app.config.get("SECURITY_TOTP_SECRETS", None) - if totp_secrets is not None: - return - totp_secrets = os.environ.get("SECURITY_TOTP_SECRETS", None) - if totp_secrets is not None: - try: - # Try to parse the string from the environment variable into a dictionary - app.config["SECURITY_TOTP_SECRETS"] = json.loads(totp_secrets) - return - except json.JSONDecodeError: - app.logger.error( - "Error: The environment variable SECURITY_TOTP_SECRETS is not valid JSON." - ) - sys.exit(2) - - filename = os.path.join(app.instance_path, filename) - try: - security_totp_secrets = open(filename, "r").read() - security_totp_secrets = json.loads(security_totp_secrets) - - # Now check if it's a dictionary - if isinstance(security_totp_secrets, dict): - app.config["SECURITY_TOTP_SECRETS"] = security_totp_secrets - else: - log_totp_secrets_error_and_exit(app, filename) - except json.JSONDecodeError: - log_totp_secrets_error_and_exit(app, filename) - except (IOError, UnicodeDecodeError): - app.logger.error( - """ - Error: No SECURITY_TOTP_SECRETS set (required for two-factor authentication). - - You can add the SECURITY_TOTP_SECRETS setting to your conf file (this example works only on Unix): - - echo "SECURITY_TOTP_SECRETS={\\"1\\": \\"`python3 -c 'from passlib import totp; print(totp.generate_secret())'`\\"}" >> ~/.flexmeasures.cfg - - OR you can add an env var: - - export SECURITY_TOTP_SECRETS={"1": "xxxxxxxxxxxxxxx"} - (on windows, use "set" instead of "export") - - OR you can create a secret key file (this example works only on Unix): - - mkdir -p %s - echo "{\\"1\\": \\"$(head -c 24 /dev/urandom | base64)\\"}" > %s - - You can also use Python to create a good secret: - - python -c 'from passlib import totp; print(f"{{\\"1\\": \\"{totp.generate_secret()}\\"}}")' - - """ - % (os.path.dirname(filename), filename) - ) - sys.exit(2) - - def root_dispatcher(): """ Re-routes to root views fitting for the current user, diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 4cbcc26cd3..e4d869054b 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -72,6 +72,7 @@ class Config(object): SECURITY_TOKEN_MAX_AGE: int = 60 * 60 * 6 # six hours SECURITY_TRACKABLE: bool = False # this is more in line with modern privacy law SECURITY_PASSWORD_SALT: str | None = None + FLEXMEASURES_SECRETS_ENCRYPTION_KEYS: dict[str, str] | None = None # Two Factor Authentication SECURITY_TWO_FACTOR_ENABLED_METHODS = [ @@ -212,8 +213,8 @@ class Config(object): # names of settings which cannot be None -# SECRET_KEY is also required but utils.app_utils.set_secret_key takes care of this better. -# SECURITY_TOTP_SECRETS is also required but utils.app_utils.set_totp_secrets takes care of this better. +# SECRET_KEY is also required but utils.secrets_utils.set_secret_key takes care of this better. +# SECURITY_TOTP_SECRETS is also required but utils.secrets_utils.set_totp_secrets takes care of this better. required: list[str] = ["SQLALCHEMY_DATABASE_URI"] # settings whose absence should trigger a warning diff --git a/flexmeasures/utils/secrets_utils.py b/flexmeasures/utils/secrets_utils.py new file mode 100644 index 0000000000..dc2f4b9133 --- /dev/null +++ b/flexmeasures/utils/secrets_utils.py @@ -0,0 +1,720 @@ +from __future__ import annotations + +import base64 +import json +import os +import sys +from copy import deepcopy +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, TypedDict + +from cryptography.fernet import Fernet, InvalidToken +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from flask import current_app + +from flexmeasures.utils.time_utils import as_utc_isoformat, server_now + +if TYPE_CHECKING: + from flexmeasures.data.models.generic_assets import GenericAsset + from flexmeasures.data.models.user import Account + + +class SecretsError(Exception): + """Raised when secret handling fails.""" + + +class InvalidSecretsEncryptionKey(SecretsError): + """Raised when a usable encryption key cannot be loaded.""" + + +class SecretsDecryptionError(SecretsError): + """Raised when a value cannot be decrypted.""" + + +_KDF_OUTPUT_BYTES = 32 +_KDF_INFO = b"flexmeasures:connection-secrets:v1" +_CIPHERTEXT_FIELD = "ciphertext" +_CONNECTION_SECRETS_KEY_VALUE_GENERATOR = ( + "import secrets; print(secrets.token_urlsafe(32))" +) +_CONNECTION_SECRETS_SETTING_GENERATOR = ( + 'import json, secrets; print(json.dumps({"1": secrets.token_urlsafe(32)}))' +) +_TOTP_SECRETS_KEY_VALUE_GENERATOR = ( + "from passlib import totp; print(totp.generate_secret())" +) +_TOTP_SECRETS_SETTING_GENERATOR = 'import json; from passlib import totp; print(json.dumps({"1": totp.generate_secret()}))' + + +class _SecretOverviewRequired(TypedDict): + path: str + + +class SecretOverview(_SecretOverviewRequired, total=False): + expires_at: datetime + + +def format_keyring_config_help( + setting_name: str, + *, + purpose: str, + filename: str | None = None, + key_value_generator_python: str = _CONNECTION_SECRETS_KEY_VALUE_GENERATOR, + setting_generator_python: str = _CONNECTION_SECRETS_SETTING_GENERATOR, +) -> str: + """Return instructions for configuring a dictionary-based secret setting. + + :param setting_name: Name of the FlexMeasures configuration setting. + :param purpose: Short explanation of what the setting protects. + :param filename: Optional instance-path file where the setting can be stored. + :param key_value_generator_python: Python snippet which prints one secret. + :param setting_generator_python: Python snippet which prints the JSON setting. + """ + file_instructions = "" + if filename is not None: + file_instructions = f""" + + OR you can create a secret key file (this example works only on Unix): + + mkdir -p {os.path.dirname(filename)} + echo "{{\\"1\\": \\"$(python3 -c '{key_value_generator_python}')\\"}}" > {filename} + """ + return f""" + Error: No {setting_name} set ({purpose}). + + Configure {setting_name} as a JSON dictionary from key IDs to secret values. + + You can add the {setting_name} setting to your conf file (this example works only on Unix): + + echo "{setting_name}={{\\"1\\": \\"`python3 -c '{key_value_generator_python}'`\\"}}" >> ~/.flexmeasures.cfg + + OR you can add an env var: + + export {setting_name}='{{"1": "xxxxxxxxxxxxxxx"}}' + (on windows, use "set" instead of "export") + {file_instructions} + + You can also use Python to create a good setting value: + + python3 -c '{setting_generator_python}' + + """ + + +def set_totp_secrets(app, filename: str = "totp_secrets") -> None: + """Set the ``SECURITY_TOTP_SECRETS`` setting or exit app startup. + + :param app: Flask app whose config should receive the setting. + :param filename: File name in the app instance path to check after config + and environment values. + """ + setting_name = "SECURITY_TOTP_SECRETS" + purpose = "required for two-factor authentication" + + if app.config.get(setting_name, None) is not None: + return + configured_value = os.environ.get(setting_name, None) + if configured_value is not None: + try: + app.config[setting_name] = json.loads(configured_value) + return + except json.JSONDecodeError: + app.logger.error( + f"Error: The environment variable {setting_name} is not valid JSON." + ) + sys.exit(2) + + path = os.path.join(app.instance_path, filename) if filename else None + if path is not None: + try: + with open(path) as keyring_file: + configured_value = json.loads(keyring_file.read()) + if isinstance(configured_value, dict): + app.config[setting_name] = configured_value + return + log_keyring_config_error_and_exit(app, setting_name, path) + except json.JSONDecodeError: + log_keyring_config_error_and_exit(app, setting_name, path) + except (IOError, UnicodeDecodeError): + pass + + app.logger.error( + format_keyring_config_help( + setting_name, + purpose=purpose, + filename=path, + key_value_generator_python=_TOTP_SECRETS_KEY_VALUE_GENERATOR, + setting_generator_python=_TOTP_SECRETS_SETTING_GENERATOR, + ) + ) + sys.exit(2) + + +def log_keyring_config_error_and_exit(app, setting_name: str, filename: str) -> None: + """Log invalid keyring-file instructions and exit app startup. + + :param app: Flask app whose logger should receive the message. + :param setting_name: Name of the FlexMeasures configuration setting. + :param filename: File path containing an invalid value. + """ + app.logger.error( + """ + ERROR: The file %s exists but does not contain a valid dictionary for %s. + + The correct format is: + + {"1": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"} + + """ + % (filename, setting_name) + ) + sys.exit(2) + + +def set_secret_key(app, filename: str = "secret_key") -> None: + """Set the ``SECRET_KEY`` setting or exit app startup. + + :param app: Flask app whose config should receive the setting. + :param filename: File name in the app instance path to check after config + and environment values. + """ + secret_key = app.config.get("SECRET_KEY", None) + if secret_key is not None: + return + secret_key = os.environ.get("SECRET_KEY", None) + if secret_key is not None: + app.config["SECRET_KEY"] = secret_key + return + filename = os.path.join(app.instance_path, filename) + try: + with open(filename, "rb") as secret_key_file: + app.config["SECRET_KEY"] = secret_key_file.read() + except IOError: + app.logger.error( + """ + Error: No secret key set. + + You can add the SECRET_KEY setting to your conf file (this example works only on Unix): + + echo "SECRET_KEY=\\"`python3 -c 'import secrets; print(secrets.token_hex(24))'`\\"" >> ~/.flexmeasures.cfg + + OR you can add an env var: + + export SECRET_KEY=xxxxxxxxxxxxxxx + (on windows, use "set" instead of "export") + + OR you can create a secret key file (this example works only on Unix): + + mkdir -p %s + head -c 24 /dev/urandom > %s + + You can also use Python to create a good secret: + + python3 -c "import secrets; print(secrets.token_urlsafe())" + + """ + % (os.path.dirname(filename), filename) + ) + + sys.exit(2) + + +def derive_fernet_key(secret: str, *, salt: str = "flexmeasures-secrets") -> bytes: + """Derive a Fernet-compatible key from an arbitrary non-empty secret. + + :param secret: Master secret from configuration. + :param salt: Context-specific salt for key derivation. + :return: URL-safe base64-encoded key accepted by Fernet. + """ + if not isinstance(secret, str) or not secret.strip(): + raise InvalidSecretsEncryptionKey("Secret encryption key must be non-empty.") + + hkdf = HKDF( + algorithm=hashes.SHA256(), + length=_KDF_OUTPUT_BYTES, + salt=salt.encode("utf-8"), + info=_KDF_INFO, + ) + derived = hkdf.derive(secret.encode("utf-8")) + return base64.urlsafe_b64encode(derived) + + +@dataclass(frozen=True, slots=True) +class SecretsEncryptor: + """Encrypt and decrypt connection secrets with the configured master key. + + :param encryption_keys: Mapping from key IDs to master key material. + :param key_id: Identifier of the key used for new encryption. If empty, the + latest key ID in ``encryption_keys`` is used. + """ + + encryption_keys: dict[str, str] + key_id: str = "" + + def __post_init__(self) -> None: + if not isinstance(self.encryption_keys, dict): + raise InvalidSecretsEncryptionKey( + "Secret encryption keys must be a non-empty dictionary." + ) + encryption_keys = _normalize_keyring(self.encryption_keys) + key_id = self.key_id if self.key_id else _latest_key_id(encryption_keys) + if not encryption_keys: + raise InvalidSecretsEncryptionKey( + "Secret encryption keys must contain at least one non-empty string." + ) + if not isinstance(key_id, str) or not key_id.strip(): + raise InvalidSecretsEncryptionKey("Secret encryption key ID is required.") + if key_id not in encryption_keys: + raise InvalidSecretsEncryptionKey( + f"Secret encryption key {key_id!r} is not configured." + ) + object.__setattr__(self, "encryption_keys", encryption_keys) + object.__setattr__(self, "key_id", key_id) + + @classmethod + def from_current_app(cls) -> "SecretsEncryptor": + """Create an encryptor from Flask configuration. + + ``FLEXMEASURES_SECRETS_ENCRYPTION_KEYS`` must be configured before + connection secrets can be stored or decrypted. + """ + encryption_keys = current_app.config.get("FLEXMEASURES_SECRETS_ENCRYPTION_KEYS") + if encryption_keys is None: + help_msg = format_keyring_config_help( + "FLEXMEASURES_SECRETS_ENCRYPTION_KEYS", + purpose="required before storing connection secrets", + key_value_generator_python=_CONNECTION_SECRETS_KEY_VALUE_GENERATOR, + setting_generator_python=_CONNECTION_SECRETS_SETTING_GENERATOR, + ) + raise InvalidSecretsEncryptionKey(help_msg) + if not isinstance(encryption_keys, dict) or not encryption_keys: + raise InvalidSecretsEncryptionKey( + "FLEXMEASURES_SECRETS_ENCRYPTION_KEYS must be a non-empty dictionary." + ) + return cls(encryption_keys=encryption_keys) + + @property + def fernet(self) -> Fernet: + """Fernet instance derived from the current master key.""" + return self._fernet_for(self.key_id) + + def _fernet_for(self, key_id: str) -> Fernet: + try: + encryption_key = self.encryption_keys[key_id] + except KeyError as exc: + raise InvalidSecretsEncryptionKey( + f"Secret encryption key {key_id!r} is not configured." + ) from exc + return Fernet(derive_fernet_key(encryption_key)) + + def encrypt(self, value: str) -> dict[str, Any]: + """Encrypt a string and return a JSON-serializable envelope. + + :param value: Plaintext secret value to encrypt. + :return: Dict containing ciphertext, key ID and timestamps. + """ + if not isinstance(value, str): + raise SecretsError("Only string secret values can be encrypted.") + now = as_utc_isoformat(server_now()) + return { + _CIPHERTEXT_FIELD: self.fernet.encrypt(value.encode("utf-8")).decode( + "utf-8" + ), + "key_id": self.key_id, + "created_at": now, + "updated_at": now, + } + + def decrypt(self, envelope: dict[str, Any] | str) -> str: + """Decrypt an encrypted envelope or raw Fernet token. + + :param envelope: Dict with a ``ciphertext`` field, or a raw token. + :return: Decrypted plaintext value. + """ + token = ( + envelope.get(_CIPHERTEXT_FIELD) if isinstance(envelope, dict) else envelope + ) + if not isinstance(token, str) or not token: + raise SecretsDecryptionError("Secret envelope does not contain ciphertext.") + if isinstance(envelope, dict) and isinstance(envelope.get("key_id"), str): + key_ids = [envelope["key_id"]] + else: + key_ids = [ + self.key_id, + *[key for key in self.encryption_keys if key != self.key_id], + ] + for key_id in key_ids: + try: + raw = self._fernet_for(key_id).decrypt(token.encode("utf-8")) + except InvalidToken: + continue + return raw.decode("utf-8") + raise SecretsDecryptionError("Invalid secret token.") + + +def _normalize_keyring(encryption_keys: dict[str, str]) -> dict[str, str]: + """Return a copy of the keyring with non-empty string values only.""" + normalized_keys = { + str(key_id): key + for key_id, key in encryption_keys.items() + if isinstance(key, str) and key.strip() + } + if not normalized_keys: + raise InvalidSecretsEncryptionKey( + "FLEXMEASURES_SECRETS_ENCRYPTION_KEYS must contain non-empty string values." + ) + return normalized_keys + + +def _latest_key_id(encryption_keys: dict[str, str]) -> str: + """Return the latest key ID from a keyring (dictionary of encryption keys).""" + numeric_key_ids = [ + int(key_id) for key_id in encryption_keys if str(key_id).isdigit() + ] + if numeric_key_ids: + return str(max(numeric_key_ids)) + return sorted(encryption_keys)[-1] + + +def _path_parts(path: str | tuple[str, ...] | list[str]) -> list[str]: + """Return one or two non-empty string parts from a secret path.""" + if isinstance(path, str): + parts = [part for part in path.split(".") if part] + else: + parts = list(path) + if not parts or any(not isinstance(part, str) or not part for part in parts): + raise ValueError("Secret path must contain at least one non-empty string part.") + if len(parts) > 2: + raise ValueError( + "Secret path must contain at most two parts. " + "Use a tuple or list if a secret name itself contains dots." + ) + return parts + + +def _is_secret_envelope(value: Any) -> bool: + """Return whether ``value`` looks like an encrypted secret envelope.""" + return isinstance(value, dict) and _CIPHERTEXT_FIELD in value + + +def set_secret( + secrets: dict[str, Any] | None, + path: str | tuple[str, ...] | list[str], + value: str, + *, + encryptor: SecretsEncryptor | None = None, + metadata: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Return a copy of ``secrets`` with an encrypted value set at ``path``. + + :param secrets: Existing secrets dictionary, or ``None`` for a new one. + :param path: Dot-separated path or sequence of keys where the value is stored. + :param value: Plaintext secret value to encrypt. + :param encryptor: Optional encryptor; defaults to app configuration. + :param metadata: Optional non-secret metadata to store with the envelope. + """ + encryptor = encryptor or SecretsEncryptor.from_current_app() + updated = deepcopy(secrets or {}) + parts = _path_parts(path) + envelope = encryptor.encrypt(value) + if metadata: + envelope.update(metadata) + if len(parts) == 1: + existing = updated.get(parts[0]) + if ( + isinstance(existing, dict) + and not _is_secret_envelope(existing) + and existing + ): + raise ValueError( + f"Secret path {parts[0]} conflicts with nested secrets stored under the same name." + ) + updated[parts[0]] = envelope + return updated + + namespace, secret_name = parts + existing_namespace = updated.get(namespace) + if _is_secret_envelope(existing_namespace): + raise ValueError( + f"Secret path {namespace}.{secret_name} conflicts with a secret already stored at {namespace}." + ) + if existing_namespace is None: + updated[namespace] = {} + existing_namespace = updated[namespace] + if not isinstance(existing_namespace, dict): + raise ValueError(f"Secret path conflicts with non-object value at {namespace}.") + existing_namespace[secret_name] = envelope + return updated + + +def delete_secret( + secrets: dict[str, Any] | None, + path: str | tuple[str, ...] | list[str], +) -> dict[str, Any]: + """Return a copy of ``secrets`` without the value at ``path``. + + Empty dictionaries containing the deleted value are removed as well. + + :param secrets: Existing secrets dictionary. + :param path: Dot-separated path or sequence of keys to remove. + :raises KeyError: If the path does not exist. + """ + updated = deepcopy(secrets or {}) + current: Any = updated + parents: list[tuple[dict[str, Any], str]] = [] + parts = _path_parts(path) + for part in parts[:-1]: + if not isinstance(current, dict) or part not in current: + raise KeyError("Secret path does not exist.") + parents.append((current, part)) + current = current[part] + if not isinstance(current, dict) or parts[-1] not in current: + raise KeyError("Secret path does not exist.") + del current[parts[-1]] + + for parent, part in reversed(parents): + child = parent[part] + if isinstance(child, dict) and not child: + del parent[part] + else: + break + return updated + + +def store_account_secret( + account: "Account", + path: str | tuple[str, ...] | list[str], + value: str, + *, + metadata: dict[str, Any] | None = None, + encryptor: SecretsEncryptor | None = None, +) -> dict[str, Any]: + """Store an encrypted secret on an account and return its secrets dict. + + :param account: Account whose ``secrets`` field should be updated. + :param path: Dot-separated path or sequence of keys where the value is stored. + :param value: Plaintext secret value to encrypt. + :param metadata: Optional non-secret metadata to store with the envelope. + :param encryptor: Optional encryptor; defaults to app configuration. + """ + account.secrets = set_secret( + account.secrets, + path, + value, + metadata=metadata, + encryptor=encryptor, + ) + return account.secrets + + +def store_asset_secret( + asset: "GenericAsset", + path: str | tuple[str, ...] | list[str], + value: str, + *, + metadata: dict[str, Any] | None = None, + encryptor: SecretsEncryptor | None = None, +) -> dict[str, Any]: + """Store an encrypted secret on an asset and return its secrets dict. + + :param asset: Generic asset whose ``secrets`` field should be updated. + :param path: Dot-separated path or sequence of keys where the value is stored. + :param value: Plaintext secret value to encrypt. + :param metadata: Optional non-secret metadata to store with the envelope. + :param encryptor: Optional encryptor; defaults to app configuration. + """ + asset.secrets = set_secret( + asset.secrets, + path, + value, + metadata=metadata, + encryptor=encryptor, + ) + return asset.secrets + + +def get_secret( + secrets: dict[str, Any] | None, + path: str | tuple[str, ...] | list[str], + *, + encryptor: SecretsEncryptor | None = None, +) -> str: + """Decrypt and return a secret value from ``path``. + + :param secrets: Secrets dictionary containing encrypted envelopes. + :param path: Dot-separated path or sequence of keys to the encrypted value. + :param encryptor: Optional encryptor; defaults to app configuration. + :return: Decrypted plaintext value. + """ + encryptor = encryptor or SecretsEncryptor.from_current_app() + current: Any = secrets or {} + parts = _path_parts(path) + for part in parts: + if not isinstance(current, dict) or part not in current: + raise KeyError("Secret path does not exist.") + current = current[part] + if not _is_secret_envelope(current): + raise KeyError("Secret path does not exist.") + return encryptor.decrypt(current) + + +def get_secret_paths(secrets: dict[str, Any] | None) -> list[str]: + """Return sorted dot-separated paths to encrypted secret values. + + :param secrets: Secrets dictionary containing encrypted envelopes. + :return: Secret paths without ciphertext or metadata values. + """ + return [secret["path"] for secret in get_secret_overview(secrets)] + + +def get_secret_overview( + secrets: dict[str, Any] | None, +) -> list[SecretOverview]: + """Return safe information for listing stored secrets. + + :param secrets: Secrets dictionary containing encrypted envelopes. + :return: Secret paths with optional expiry datetimes, without other metadata. + """ + overview: list[SecretOverview] = [] + + def _append_overview(path: str, value: dict[str, Any]) -> None: + secret: SecretOverview = {"path": path} + expires_at = value.get("expires_at") + if isinstance(expires_at, str): + try: + parsed_expires_at = datetime.fromisoformat( + f"{expires_at[:-1]}+00:00" + if expires_at.endswith("Z") + else expires_at + ) + except ValueError: + pass + else: + if ( + parsed_expires_at.tzinfo is None + or parsed_expires_at.utcoffset() is None + ): + parsed_expires_at = parsed_expires_at.replace(tzinfo=timezone.utc) + secret["expires_at"] = parsed_expires_at + overview.append(secret) + + for key, value in (secrets or {}).items(): + if _is_secret_envelope(value): + _append_overview(key, value) + continue + if not isinstance(value, dict): + continue + for nested_key, nested_value in value.items(): + if _is_secret_envelope(nested_value): + _append_overview(f"{key}.{nested_key}", nested_value) + + return sorted(overview, key=lambda secret: secret["path"]) + + +def redact_secrets(secrets: dict[str, Any] | None) -> dict[str, Any]: + """Return secrets metadata without ciphertext or plaintext values. + + :param secrets: Secrets dictionary to redact. + :return: Copy with encrypted values replaced by safe metadata. + """ + if not secrets: + return {} + + def _redact(value: Any) -> Any: + if isinstance(value, dict): + if _CIPHERTEXT_FIELD in value: + return { + "set": bool(value.get(_CIPHERTEXT_FIELD)), + **{ + key: redacted_value + for key in ( + "key_id", + "created_at", + "updated_at", + "expires_at", + "token_type", + ) + if (redacted_value := value.get(key)) is not None + }, + } + return {key: _redact(nested) for key, nested in value.items()} + return value + + return _redact(secrets) + + +@dataclass(frozen=True, slots=True) +class TokenRefreshResult: + """Provider-neutral result of refreshing token state. + + :param access_token: New access token, if the provider returned one. + :param refresh_token: New refresh token, if it rotated. + :param access_token_expires_at: Expiry timestamp for the access token. + :param refresh_token_expires_at: Expiry timestamp for the refresh token. + :param token_type: Token type metadata, for example ``Bearer``. + :param metadata: Provider-specific non-secret metadata to preserve. + """ + + access_token: str | None = None + refresh_token: str | None = None + access_token_expires_at: datetime | None = None + refresh_token_expires_at: datetime | None = None + token_type: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +def apply_token_refresh_result( + secrets: dict[str, Any] | None, + namespace: str, + result: TokenRefreshResult, + *, + encryptor: SecretsEncryptor | None = None, +) -> dict[str, Any]: + """Apply a provider-specific token refresh result to a secrets dictionary. + + If a token value is ``None``, only its metadata is updated. This supports + providers whose refresh operation extends an existing token instead of + returning a replacement token. + + :param secrets: Existing secrets dictionary, or ``None`` for a new one. + :param namespace: Top-level provider or strategy namespace to update. + :param result: Token values and metadata returned by a provider strategy. + :param encryptor: Optional encryptor; defaults to app configuration. + :return: Updated copy of the secrets dictionary. + """ + encryptor = encryptor or SecretsEncryptor.from_current_app() + updated = deepcopy(secrets or {}) + provider_state = updated.setdefault(namespace, {}) + if not isinstance(provider_state, dict): + raise ValueError(f"Secret namespace {namespace} conflicts with a non-object.") + + token_updates = { + "access_token": ( + result.access_token, + result.access_token_expires_at, + ), + "refresh_token": ( + result.refresh_token, + result.refresh_token_expires_at, + ), + } + for token_name, (token_value, expires_at) in token_updates.items(): + metadata = { + key: value + for key, value in { + "expires_at": as_utc_isoformat(expires_at), + "token_type": result.token_type, + }.items() + if value is not None + } + if token_value is not None: + provider_state[token_name] = encryptor.encrypt(token_value) + if metadata and token_name in provider_state: + provider_state[token_name].update(metadata) + provider_state[token_name]["updated_at"] = as_utc_isoformat(server_now()) + + if result.metadata: + provider_state.setdefault("metadata", {}).update(result.metadata) + return updated diff --git a/flexmeasures/utils/tests/test_secrets_utils.py b/flexmeasures/utils/tests/test_secrets_utils.py new file mode 100644 index 0000000000..90501897be --- /dev/null +++ b/flexmeasures/utils/tests/test_secrets_utils.py @@ -0,0 +1,504 @@ +from datetime import datetime, timezone + +import pytest + +from flexmeasures.utils.secrets_utils import ( + SecretsDecryptionError, + SecretsEncryptor, + InvalidSecretsEncryptionKey, + TokenRefreshResult, + apply_token_refresh_result, + delete_secret, + derive_fernet_key, + format_keyring_config_help, + get_secret, + get_secret_overview, + get_secret_paths, + redact_secrets, + set_secret, + set_totp_secrets, + store_account_secret, + store_asset_secret, +) + + +def test_derive_fernet_key_accepts_non_fernet_secret(): + key = derive_fernet_key("not-a-fernet-key") + + assert isinstance(key, bytes) + assert len(key) == 44 + + +def test_encrypt_decrypt_and_redact_secret(): + encryptor = SecretsEncryptor({"test-key": "test-master-key"}, key_id="test-key") + secrets = set_secret( + {}, + "connection.refresh_token", + "refresh-token-value", + encryptor=encryptor, + metadata={"expires_at": "2026-06-11T12:00:00+00:00"}, + ) + + envelope = secrets["connection"]["refresh_token"] + assert envelope["ciphertext"] != "refresh-token-value" + assert get_secret(secrets, "connection.refresh_token", encryptor=encryptor) == ( + "refresh-token-value" + ) + assert redact_secrets(secrets) == { + "connection": { + "refresh_token": { + "set": True, + "key_id": "test-key", + "created_at": envelope["created_at"], + "updated_at": envelope["updated_at"], + "expires_at": "2026-06-11T12:00:00+00:00", + } + } + } + + +def test_store_account_secret_updates_account_secrets(setup_accounts): + encryptor = SecretsEncryptor({"1": "test-master-key"}) + account = setup_accounts["Prosumer"] + + store_account_secret( + account, + "platform.refresh_token", + "refresh-token-value", + metadata={"expires_at": "2026-06-11T12:00:00+00:00"}, + encryptor=encryptor, + ) + + envelope = account.secrets["platform"]["refresh_token"] + assert envelope["ciphertext"] != "refresh-token-value" + assert envelope["expires_at"] == "2026-06-11T12:00:00+00:00" + assert get_secret( + account.secrets, "platform.refresh_token", encryptor=encryptor + ) == ("refresh-token-value") + + +def test_store_asset_secret_updates_asset_secrets(setup_generic_assets): + encryptor = SecretsEncryptor({"1": "test-master-key"}) + asset = setup_generic_assets["test_battery"] + + store_asset_secret( + asset, + "platform.password", + "password-value", + encryptor=encryptor, + ) + + envelope = asset.secrets["platform"]["password"] + assert envelope["ciphertext"] != "password-value" + assert get_secret(asset.secrets, "platform.password", encryptor=encryptor) == ( + "password-value" + ) + + +def test_decrypt_rejects_wrong_key(): + secrets = set_secret( + {}, + "connection.password", + "secret", + encryptor=SecretsEncryptor({"1": "first-key"}), + ) + + with pytest.raises(SecretsDecryptionError): + get_secret( + secrets, + "connection.password", + encryptor=SecretsEncryptor({"1": "second-key"}), + ) + + +def test_set_secret_rejects_paths_deeper_than_two_parts(): + encryptor = SecretsEncryptor({"1": "test-master-key"}) + + with pytest.raises( + ValueError, + match="at most two parts", + ): + set_secret( + {}, + "platform.tokens.refresh", + "secret", + encryptor=encryptor, + ) + + +def test_set_secret_accepts_sequence_paths_with_dots_in_secret_name(): + encryptor = SecretsEncryptor({"1": "test-master-key"}) + + secrets = set_secret( + {}, + ("platform", "token.v2"), + "secret", + encryptor=encryptor, + ) + + assert get_secret(secrets, ("platform", "token.v2"), encryptor=encryptor) == ( + "secret" + ) + assert get_secret_paths(secrets) == ["platform.token.v2"] + + +def test_set_secret_rejects_storing_nested_secret_under_existing_secret(): + encryptor = SecretsEncryptor({"1": "test-master-key"}) + secrets = set_secret({}, "platform", "root-secret", encryptor=encryptor) + + with pytest.raises( + ValueError, + match="conflicts with a secret already stored at platform", + ): + set_secret(secrets, "platform.refresh_token", "secret", encryptor=encryptor) + + +def test_set_secret_rejects_storing_root_secret_over_existing_namespace(): + encryptor = SecretsEncryptor({"1": "test-master-key"}) + secrets = set_secret({}, "platform.refresh_token", "secret", encryptor=encryptor) + + with pytest.raises( + ValueError, + match="conflicts with nested secrets stored under the same name", + ): + set_secret(secrets, "platform", "root-secret", encryptor=encryptor) + + +def test_delete_secret_removes_empty_parent_dicts(): + secrets = { + "platform": { + "access_token": {"ciphertext": "access"}, + "refresh_token": {"ciphertext": "refresh"}, + } + } + + secrets = delete_secret(secrets, "platform.access_token") + assert secrets == {"platform": {"refresh_token": {"ciphertext": "refresh"}}} + + secrets = delete_secret(secrets, "platform.refresh_token") + assert secrets == {} + + +def test_delete_secret_rejects_missing_path(): + secrets = {"platform": {"access_token": {"ciphertext": "access"}}} + + with pytest.raises(KeyError, match="Secret path does not exist"): + delete_secret(secrets, "platform.refresh_token") + + +def test_get_secret_paths_returns_only_encrypted_value_paths(): + secrets = { + "platform": { + "access_token": {"ciphertext": "access"}, + "metadata": {"scope": "read"}, + }, + "password": {"ciphertext": "password"}, + } + + assert get_secret_paths(secrets) == [ + "password", + "platform.access_token", + ] + + +def test_get_secret_overview_returns_sorted_strictly_allowlisted_fields(): + secrets = { + "platform": { + "refresh_token": { + "ciphertext": "refresh", + "key_id": "key-id", + "created_at": "2026-06-01T12:00:00+00:00", + "updated_at": "2026-06-02T12:00:00+00:00", + "token_type": "Bearer", + "provider_metadata": {"scope": "read write"}, + }, + "access_token": { + "ciphertext": "access", + "expires_at": "2026-06-11T12:00:00+02:00", + "unexpected": "must-not-be-exposed", + }, + } + } + + overview = get_secret_overview(secrets) + + assert overview == [ + { + "path": "platform.access_token", + "expires_at": datetime.fromisoformat("2026-06-11T12:00:00+02:00"), + }, + {"path": "platform.refresh_token"}, + ] + assert all(set(secret) <= {"path", "expires_at"} for secret in overview) + + +def test_get_secret_overview_lists_only_supported_one_or_two_level_secrets(): + secrets = { + "platform": { + "refresh_token": { + "ciphertext": "refresh", + "nested": { + "swallowed-before": {"ciphertext": "should-not-appear"}, + }, + }, + "metadata": {"scope": "read write"}, + }, + "standalone": { + "ciphertext": "standalone", + "expires_at": "2026-06-11T12:00:00+00:00", + }, + } + + assert get_secret_overview(secrets) == [ + { + "path": "platform.refresh_token", + }, + { + "path": "standalone", + "expires_at": datetime.fromisoformat("2026-06-11T12:00:00+00:00"), + }, + ] + + +@pytest.mark.parametrize( + ("expires_at", "expected"), + [ + ( + "2026-06-11T12:00:00+02:00", + datetime.fromisoformat("2026-06-11T12:00:00+02:00"), + ), + ( + "2026-06-11T12:00:00", + datetime(2026, 6, 11, 12, tzinfo=timezone.utc), + ), + ( + "2026-06-11T10:00:00Z", + datetime(2026, 6, 11, 10, tzinfo=timezone.utc), + ), + ], +) +def test_get_secret_overview_accepts_aware_iso_expiry_timestamps(expires_at, expected): + secrets = { + "platform": { + "access_token": { + "ciphertext": "access", + "expires_at": expires_at, + } + } + } + + overview = get_secret_overview(secrets) + + assert overview == [ + { + "path": "platform.access_token", + "expires_at": expected, + } + ] + assert overview[0]["expires_at"].tzinfo is not None + + +@pytest.mark.parametrize( + "expires_at", + ["not-a-datetime", 1781179200, None], +) +def test_get_secret_overview_omits_invalid_expiry(expires_at): + secrets = { + "platform": { + "access_token": { + "ciphertext": "access", + "expires_at": expires_at, + } + } + } + + assert get_secret_overview(secrets) == [{"path": "platform.access_token"}] + + +def test_get_secret_paths_remains_backward_compatible_with_secret_overview(): + secrets = { + "z": {"secret": {"ciphertext": "z", "expires_at": "invalid"}}, + "a": { + "secret": { + "ciphertext": "a", + "expires_at": "2026-06-11T10:00:00Z", + } + }, + } + + assert get_secret_paths(secrets) == ["a.secret", "z.secret"] + assert get_secret_paths(secrets) == [ + secret["path"] for secret in get_secret_overview(secrets) + ] + + +def test_from_current_app_requires_keyring_outside_production(app): + app.config["FLEXMEASURES_ENV"] = "testing" + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = None + app.config["SECRET_KEY"] = "testing-secret-key" + + with pytest.raises( + InvalidSecretsEncryptionKey, + match="No FLEXMEASURES_SECRETS_ENCRYPTION_KEYS set", + ): + SecretsEncryptor.from_current_app() + + +def test_set_secret_requires_configured_keyring(app): + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = None + + with pytest.raises( + InvalidSecretsEncryptionKey, + match="key IDs to secret values", + ): + set_secret({}, "connection.password", "secret") + + +def test_format_keyring_config_help_mentions_setting_and_generator(): + help_text = format_keyring_config_help( + "FLEXMEASURES_SECRETS_ENCRYPTION_KEYS", + purpose="required before storing connection secrets", + ) + + assert "FLEXMEASURES_SECRETS_ENCRYPTION_KEYS" in help_text + assert '{"1": "xxxxxxxxxxxxxxx"}' in help_text + assert "python3 -c" in help_text + assert "secrets.token_urlsafe" in help_text + + +def test_set_totp_secrets_reads_environment(app, monkeypatch): + app.config["SECURITY_TOTP_SECRETS"] = None + monkeypatch.setenv("SECURITY_TOTP_SECRETS", '{"1": "totp-secret"}') + + set_totp_secrets(app) + + assert app.config["SECURITY_TOTP_SECRETS"] == {"1": "totp-secret"} + + +def test_set_totp_secrets_reads_file(app, tmp_path): + app.config["SECURITY_TOTP_SECRETS"] = None + secret_file = tmp_path / "totp_secrets" + secret_file.write_text('{"1": "totp-secret"}') + + set_totp_secrets(app, filename=str(secret_file)) + + assert app.config["SECURITY_TOTP_SECRETS"] == {"1": "totp-secret"} + + +def test_from_current_app_uses_latest_key_from_keyring(app): + app.config["FLEXMEASURES_ENV"] = "production" + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = { + "1": "old-secret-key", + "2": "current-secret-key", + } + + encryptor = SecretsEncryptor.from_current_app() + + assert encryptor.encryption_keys == { + "1": "old-secret-key", + "2": "current-secret-key", + } + assert encryptor.key_id == "2" + + +def test_from_current_app_uses_lexical_latest_key_from_keyring(app): + app.config["FLEXMEASURES_ENV"] = "production" + app.config["FLEXMEASURES_SECRETS_ENCRYPTION_KEYS"] = { + "2026-01": "old-secret-key", + "2026-06": "current-secret-key", + } + + encryptor = SecretsEncryptor.from_current_app() + + assert encryptor.key_id == "2026-06" + + +def test_decrypt_uses_key_id_from_envelope(): + old_encryptor = SecretsEncryptor( + {"1": "old-secret-key", "2": "current-secret-key"}, + key_id="1", + ) + current_encryptor = SecretsEncryptor( + {"1": "old-secret-key", "2": "current-secret-key"}, + key_id="2", + ) + + envelope = old_encryptor.encrypt("old-secret-value") + + assert current_encryptor.decrypt(envelope) == "old-secret-value" + + +def test_decrypt_raw_token_can_try_all_configured_keys(): + old_encryptor = SecretsEncryptor( + {"1": "old-secret-key", "2": "current-secret-key"}, + key_id="1", + ) + current_encryptor = SecretsEncryptor( + {"1": "old-secret-key", "2": "current-secret-key"}, + key_id="2", + ) + raw_token = old_encryptor.encrypt("old-secret-value")["ciphertext"] + + assert current_encryptor.decrypt(raw_token) == "old-secret-value" + + +def test_apply_token_refresh_result_can_update_tokens_and_metadata(): + encryptor = SecretsEncryptor({"1": "token-key"}) + result = TokenRefreshResult( + access_token="access-1", + refresh_token="refresh-1", + access_token_expires_at=datetime(2026, 6, 11, 12, 5, tzinfo=timezone.utc), + refresh_token_expires_at=datetime(2026, 6, 25, 12, 0, tzinfo=timezone.utc), + token_type="Bearer", + metadata={"scope": "read write"}, + ) + + secrets = apply_token_refresh_result( + {}, + "platform", + result, + encryptor=encryptor, + ) + + assert get_secret(secrets, "platform.access_token", encryptor=encryptor) == ( + "access-1" + ) + assert get_secret(secrets, "platform.refresh_token", encryptor=encryptor) == ( + "refresh-1" + ) + assert secrets["platform"]["access_token"]["expires_at"] == ( + "2026-06-11T12:05:00+00:00" + ) + assert secrets["platform"]["refresh_token"]["expires_at"] == ( + "2026-06-25T12:00:00+00:00" + ) + assert secrets["platform"]["metadata"] == {"scope": "read write"} + + +def test_apply_token_refresh_result_can_extend_existing_access_token(): + encryptor = SecretsEncryptor({"1": "token-key"}) + secrets = apply_token_refresh_result( + {}, + "platform", + TokenRefreshResult(access_token="existing-access-token"), + encryptor=encryptor, + ) + original_ciphertext = secrets["platform"]["access_token"]["ciphertext"] + + secrets = apply_token_refresh_result( + secrets, + "platform", + TokenRefreshResult( + access_token=None, + access_token_expires_at=datetime(2026, 6, 11, 12, 10, tzinfo=timezone.utc), + ), + encryptor=encryptor, + ) + + assert secrets["platform"]["access_token"]["ciphertext"] == original_ciphertext + assert get_secret(secrets, "platform.access_token", encryptor=encryptor) == ( + "existing-access-token" + ) + assert secrets["platform"]["access_token"]["expires_at"] == ( + "2026-06-11T12:10:00+00:00" + ) diff --git a/flexmeasures/utils/tests/test_time_utils.py b/flexmeasures/utils/tests/test_time_utils.py index d0d38632ec..e1e10a577a 100644 --- a/flexmeasures/utils/tests/test_time_utils.py +++ b/flexmeasures/utils/tests/test_time_utils.py @@ -8,6 +8,7 @@ import pytest from flexmeasures.utils.time_utils import ( + as_utc_isoformat, duration_isoformat, server_now, naturalized_datetime_str, @@ -48,6 +49,28 @@ def test_original_duration_isoformat(td: timedelta, iso: str): assert original_duration_isoformat(td) == iso +@pytest.mark.parametrize( + "dt, expected", + [ + (None, None), + ( + datetime(2026, 6, 12, 10, 30), + "2026-06-12T10:30:00+00:00", + ), + ( + datetime(2026, 6, 12, 12, 30, tzinfo=timezone.utc), + "2026-06-12T12:30:00+00:00", + ), + ( + pytz.timezone("Europe/Amsterdam").localize(datetime(2026, 6, 12, 14, 30)), + "2026-06-12T12:30:00+00:00", + ), + ], +) +def test_as_utc_isoformat(dt, expected): + assert as_utc_isoformat(dt) == expected + + @pytest.mark.parametrize( "dt_tz, now, server_tz, delta_in_h, exp_result", # there can be two results depending of today's date, due to humanize. diff --git a/flexmeasures/utils/time_utils.py b/flexmeasures/utils/time_utils.py index 382192c462..bbe8d01c6c 100644 --- a/flexmeasures/utils/time_utils.py +++ b/flexmeasures/utils/time_utils.py @@ -64,6 +64,18 @@ def naive_utc_from(dt: datetime) -> datetime: return dt.astimezone(pytz.utc).replace(tzinfo=None) +def as_utc_isoformat(dt: datetime | None) -> str | None: + """Return an ISO-formatted UTC datetime string. + + If dt is naive, assume it already represents UTC time. + """ + if dt is None: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).isoformat() + + def tz_index_naively( data: pd.DataFrame | pd.Series | pd.DatetimeIndex, ) -> pd.DataFrame | pd.Series | pd.DatetimeIndex: