diff --git a/docs/source/schemas/schema.json b/docs/source/schemas/schema.json index dedea1d..0e63fd7 100644 --- a/docs/source/schemas/schema.json +++ b/docs/source/schemas/schema.json @@ -348,7 +348,7 @@ }, "match": { "$ref": "#/definitions/DatasourcesConfig-Datasource-MARS-Match", - "description": "filters requests based on presence of specific key=value pair in user request" + "description": "filters requests based on request keys and values. Non-date keys accept scalar or list values. Date supports old-style relative rules (e.g. >30d, <40d) and MARS date rules (single/list/range/by-step)." }, "patch": { "$ref": "#/definitions/DatasourcesConfig-Datasource-MARS-Patch", @@ -455,7 +455,7 @@ }, "match": { "$ref": "#/definitions/DatasourcesConfig-Datasource-MARS-Match", - "description": "filters requests based on presence of specific key=value pair in user request" + "description": "filters requests based on request keys and values. Non-date keys accept scalar or list values. Date supports old-style relative rules (e.g. >30d, <40d) and MARS date rules (single/list/range/by-step)." }, "patch": { "$ref": "#/definitions/DatasourcesConfig-Datasource-MARS-Patch", @@ -474,26 +474,77 @@ }, "DatasourcesConfig-Datasource-MARS-Match": { "type": "object", + "description": "Datasource match rules keyed by request parameter. For non-date keys, scalar and list forms are supported. For date, old-style relative comparisons and MARS date-string rules are supported.", "properties": { + "date": { + "$ref": "#/definitions/DatasourcesConfig-Datasource-MARS-Match-DateValue", + "description": "Date matching rules. Old-style (all rules must pass): >30d, <40d. MARS-style (each requested date must match at least one rule): -1/-5/-10, -1/to/-20, -4/to/-20/by/4, 2024-02-21/to/2025-03-01/by/10.", + "examples": [ + ">30d", + [ + ">30d", + "<40d" + ], + "-1/to/-20", + [ + "-1/-5/-10", + "-20/to/-30" + ] + ] + }, "key1": { + "$ref": "#/definitions/DatasourcesConfig-Datasource-MARS-Match-Value", + "description": "Example non-date match key. Request values must be a subset of the allowed values." + }, + "key2": { + "$ref": "#/definitions/DatasourcesConfig-Datasource-MARS-Match-Value" + } + }, + "preferredOrder": [ + "date", + "key1", + "key2" + ], + "additionalProperties": { + "$ref": "#/definitions/DatasourcesConfig-Datasource-MARS-Match-Value" + } + }, + "DatasourcesConfig-Datasource-MARS-Match-Value": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { "type": "array", "items": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] }, - "default": "[]", - "description": "value matches any in list" + "default": [] + } + ] + }, + "DatasourcesConfig-Datasource-MARS-Match-DateValue": { + "oneOf": [ + { + "type": "string" }, - "key2": { + { "type": "array", "items": { "type": "string" }, - "default": "[]" + "default": [] } - }, - "preferredOrder": [ - "key1", - "key2" ] }, "DatasourcesConfig-Datasource-MARS-Patch": { diff --git a/polytope_server/common/datasource/date_check.py b/polytope_server/common/datasource/date_check.py index f853152..e4d7f40 100644 --- a/polytope_server/common/datasource/date_check.py +++ b/polytope_server/common/datasource/date_check.py @@ -3,6 +3,8 @@ from dateutil.relativedelta import relativedelta +from ..exceptions import ServerError + class DateError(Exception): """Custom exception for date-related errors.""" @@ -10,13 +12,48 @@ class DateError(Exception): pass +def date_check(date: str, allowed_values: list[str]): + """ + Process special match rules for DATE constraints. + + :param date: Date to check, can be a single, list or range of dates (Mars date format) + :param allowed_values: List of rules. All rules must be the same style: + + Old style (e.g. ">30d", "<40d"): + Each date must satisfy ALL rules (AND logic). + + New style – Mars date strings (e.g. "-1/-5/-10", "-1/to/-20", "-4/to/-20/by/4"): + Each individual date must match AT LEAST ONE rule (OR logic). + """ + if not isinstance(allowed_values, list): + raise ServerError("Allowed values must be a list") + + if not allowed_values: + return True + + if all(map(is_old_style_rule, allowed_values)): + # Old-style: every rule must pass + for rule in allowed_values: + if not date_check_single_rule(date, rule): + return False + return True + + # New-style Mars date rules: each user date must match at least one rule + user_dates = expand_mars_dates(date) + for user_date in user_dates: + if not any(date_in_mars_rule(user_date, rule) for rule in allowed_values): + raise DateError(f"Date {user_date} does not match any allowed date rule: {allowed_values}") + + return True + + def check_single_date(date, offset, offset_fmted, after=False): # Date is relative (0 = now, -1 = one day ago) if str(date)[0] == "0" or str(date)[0] == "-": date_offset = int(date) dt = datetime.today() + timedelta(days=date_offset) - if after and dt >= offset: + if after and dt > offset: raise DateError("Date is too recent, expected < {}".format(offset_fmted)) elif not after and dt < offset: raise DateError("Date is too old, expected > {}".format(offset_fmted)) @@ -28,7 +65,7 @@ def check_single_date(date, offset, offset_fmted, after=False): dt = datetime.strptime(date, "%Y%m%d") except ValueError: raise DateError("Invalid date, expected real date in YYYYMMDD format") - if after and dt >= offset: + if after and dt > offset: raise DateError("Date is too recent, expected < {}".format(offset_fmted)) elif not after and dt < offset: raise DateError("Date is too old, expected > {}".format(offset_fmted)) @@ -52,26 +89,125 @@ def parse_relativedelta(time_str): return relativedelta(days=time_dict["d"], hours=time_dict["h"], minutes=time_dict["m"]) -def date_check(date, allowed_values: list): +def is_old_style_rule(rule: str) -> bool: + """Returns True if rule is old-style (starts with > or <).""" + return rule.strip()[0] in (">", "<") + + +def parse_mars_date_token(token: str) -> datetime: + """Parse a single Mars date token to a datetime. + + Supports: + - Relative dates: 0 (today), -1 (yesterday), -10, etc. + - Absolute YYYYMMDD: 20250125 + - Absolute YYYY-MM-DD: 2023-04-23 """ - Process special match rules for DATE constraints + token = token.strip() + if not token: + raise DateError("Empty date token") + # Relative date: starts with '-' or '0' (matches existing check_single_date convention) + if token[0] == "-" or token[0] == "0": + try: + offset = int(token) + return datetime.today().replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=offset) + except ValueError: + pass + # ISO format YYYY-MM-DD + try: + return datetime.strptime(token, "%Y-%m-%d") + except ValueError: + pass + # YYYYMMDD format + try: + return datetime.strptime(token, "%Y%m%d") + except ValueError: + raise DateError(f"Invalid Mars date token: {token!r}") - :param date: Date to check, can be a string or list of strings - :param allowed_values: List of allowed values for the date in the format >1d, <2d, >1m, <2h, r"(\\d+)([dhm])". + +def expand_mars_dates(date_str: str) -> list: + """Expand a Mars date string to a list of date objects. + + Handles single dates, lists, ranges, and ranges with step. + + Examples: + "-1" -> [date(-1)] + "-1/-5/-10" -> [date(-1), date(-5), date(-10)] + "-1/to/-20" -> [date(-1), date(-2), ..., date(-20)] + "-4/to/-20/by/4" -> [date(-4), date(-8), date(-12), date(-16), date(-20)] + "20250125/-5/2023-04-23" -> [date(20250125), date(-5), date(2023-04-23)] + "2024-02-21/to/2025-03-01/by/10" -> dates every 10 days across the range """ - if not isinstance(allowed_values, list): - raise DateError("Allowed values must be a list") + parts = date_str.split("/") + + # Range syntax: second element is 'to' + if len(parts) >= 3 and parts[1].strip().lower() == "to": + start_d = parse_mars_date_token(parts[0]).date() + end_d = parse_mars_date_token(parts[2]).date() + step = 1 + if len(parts) == 5: + if parts[3].strip().lower() != "by": + raise DateError(f"Invalid Mars date string: {date_str!r}") + step = abs(int(parts[4].strip())) + elif len(parts) != 3: + raise DateError(f"Invalid Mars date string: {date_str!r}") + + dates = [] + if start_d <= end_d: + current = start_d + while current <= end_d: + dates.append(current) + current += timedelta(days=step) + else: + current = start_d + while current >= end_d: + dates.append(current) + current -= timedelta(days=step) + return dates + + # List or single date + return [parse_mars_date_token(p).date() for p in parts] + - for allowed in allowed_values: - if not date_check_single_rule(date, allowed): +def date_in_mars_rule(date_d, rule_str: str) -> bool: + """Check whether a single date (a date object) is covered by a Mars date rule string. + + The rule string follows the same Mars date syntax: + - Single: '-1', '20250125', '2023-04-23' + - List: '-1/-5/-10', '20250125/-5/2023-04-23' + - Range: '-1/to/-20', '2024-02-21/to/2025-03-01' + - Range+step: '-4/to/-20/by/4', '2024-02-21/to/2025-03-01/by/10' + """ + parts = rule_str.split("/") + + # Range syntax + if len(parts) >= 3 and parts[1].strip().lower() == "to": + start_d = parse_mars_date_token(parts[0]).date() + end_d = parse_mars_date_token(parts[2]).date() + step = 1 + if len(parts) == 5: + if parts[3].strip().lower() != "by": + raise DateError(f"Invalid Mars date rule: {rule_str!r}") + step = abs(int(parts[4].strip())) + elif len(parts) != 3: + raise DateError(f"Invalid Mars date rule: {rule_str!r}") + + min_d = min(start_d, end_d) + max_d = max(start_d, end_d) + if not (min_d <= date_d <= max_d): return False + # Date must fall on a step boundary from start + return abs((date_d - start_d).days) % step == 0 - return True + # List or single: date must equal one of the listed tokens + for part in parts: + if parse_mars_date_token(part).date() == date_d: + return True + return False def date_check_single_rule(date, allowed_values: str): """ - Process special match rules for DATE constraints + Process special match rules for DATE constraints (old-style rules only). :param date: Date to check, can be a string or list of strings :param allowed_values: Allowed values for the date in the format >1d, <2d, >1m, <2h, r"(\\d+)([dhm])". @@ -89,7 +225,7 @@ def date_check_single_rule(date, allowed_values: str): elif comp == ">": after = True else: - raise DateError(f"Invalid date comparison {comp}, expected < or >") + raise ServerError(f"Invalid date comparison {comp}, expected < or >") now = datetime.today() offset = now - parse_relativedelta(offset) offset_fmted = offset.strftime("%Y%m%d") diff --git a/tests/unit/test_date_check.py b/tests/unit/test_date_check.py new file mode 100644 index 0000000..6678b0c --- /dev/null +++ b/tests/unit/test_date_check.py @@ -0,0 +1,304 @@ +# +# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. +# + +from datetime import date, datetime, timedelta + +import pytest + +from polytope_server.common.datasource.date_check import ( + DateError, + date_check, + date_in_mars_rule, + expand_mars_dates, + parse_mars_date_token, +) + + +def d(offset): + """Return the date object for today + offset days.""" + return (datetime.today() + timedelta(days=offset)).date() + + +def ds(offset): + """Return YYYYMMDD string for today + offset days.""" + return (datetime.today() + timedelta(days=offset)).strftime("%Y%m%d") + + +# --------------------------------------------------------------------------- +# parse_mars_date_token +# --------------------------------------------------------------------------- + + +class TestParseMarsDateToken: + def test_relative_negative_one(self): + assert parse_mars_date_token("-1").date() == d(-1) + + def test_relative_negative_large(self): + assert parse_mars_date_token("-365").date() == d(-365) + + def test_relative_zero(self): + assert parse_mars_date_token("0").date() == d(0) + + def test_absolute_yyyymmdd(self): + assert parse_mars_date_token("20250125").date() == date(2025, 1, 25) + + def test_absolute_iso(self): + assert parse_mars_date_token("2023-04-23").date() == date(2023, 4, 23) + + def test_absolute_iso_leap_day(self): + assert parse_mars_date_token("2024-02-29").date() == date(2024, 2, 29) + + def test_invalid_string(self): + with pytest.raises(DateError): + parse_mars_date_token("notadate") + + def test_empty_string(self): + with pytest.raises(DateError): + parse_mars_date_token("") + + # Edge case: positive integer — NOT treated as relative (no leading 0 or -) + # "1" is an invalid YYYYMMDD and not a relative token, so it should fail. + def test_positive_integer_not_supported_as_relative(self): + with pytest.raises(DateError): + parse_mars_date_token("1") + + +# --------------------------------------------------------------------------- +# expand_mars_dates +# --------------------------------------------------------------------------- + + +class TestExpandMarsDates: + # Single dates + def test_single_relative(self): + assert expand_mars_dates("-1") == [d(-1)] + assert expand_mars_dates("0") == [d(0)] + + def test_single_absolute(self): + assert expand_mars_dates("20250125") == [date(2025, 1, 25)] + assert expand_mars_dates("2023-04-23") == [date(2023, 4, 23)] + + # Lists + def test_list_relative(self): + assert expand_mars_dates("-1/-5/-10") == [d(-1), d(-5), d(-10)] + + def test_list_mixed_formats(self): + # from the spec: "20250125/-5/2023-04-23" + result = expand_mars_dates("20250125/-5/2023-04-23") + assert result == [date(2025, 1, 25), d(-5), date(2023, 4, 23)] + + def test_list_two_elements(self): + # Two-element list must NOT be mistaken for a range + result = expand_mars_dates("-1/-20") + assert result == [d(-1), d(-20)] + assert len(result) == 2 + + # Ranges + def test_range(self): + # Ascending range + result = expand_mars_dates("2025-01-01/to/2025-01-05") + assert result == [date(2025, 1, i) for i in range(1, 6)] + # Descending range is valid: start > end in calendar terms + result = expand_mars_dates("2025-01-05/to/2025-01-01") + assert result == [date(2025, 1, i) for i in range(5, 0, -1)] + # Relative descending: -1/to/-20 + result = expand_mars_dates("-1/to/-20") + assert result[0] == d(-1) + assert result[-1] == d(-20) + assert len(result) == 20 + + # Ranges with step + def test_stepped_range(self): + # Relative: -4/to/-20/by/4 -> -4, -8, -12, -16, -20 + result = expand_mars_dates("-4/to/-20/by/4") + assert result == [d(-4), d(-8), d(-12), d(-16), d(-20)] + # Absolute: 2024-02-21/to/2025-03-01/by/10 + result = expand_mars_dates("2024-02-21/to/2025-03-01/by/10") + assert result[0] == date(2024, 2, 21) + assert all((r - date(2024, 2, 21)).days % 10 == 0 for r in result) + # Symmetric: forward and backward should produce same set + fwd = set(expand_mars_dates("-4/to/-20/by/4")) + rev = set(expand_mars_dates("-20/to/-4/by/4")) + assert fwd == rev + # Uneven step: stops before reaching exact end + result = expand_mars_dates("-4/to/-21/by/4") + assert d(-20) in result + assert d(-21) not in result + + def test_case_insensitive_to(self): + assert expand_mars_dates("-1/TO/-3") == expand_mars_dates("-1/to/-3") + + def test_case_insensitive_by(self): + assert expand_mars_dates("-4/to/-20/BY/4") == expand_mars_dates("-4/to/-20/by/4") + + +# --------------------------------------------------------------------------- +# date_in_mars_rule +# --------------------------------------------------------------------------- + + +class TestDateInMarsRule: + # Single date rules + def test_single_date(self): + assert date_in_mars_rule(d(-1), "-1") is True + assert date_in_mars_rule(d(-2), "-1") is False + assert date_in_mars_rule(date(2025, 1, 25), "20250125") is True + + # List rules + def test_list(self): + assert date_in_mars_rule(d(-1), "-1/-5/-10") is True + assert date_in_mars_rule(d(-5), "-1/-5/-10") is True + assert date_in_mars_rule(d(-10), "-1/-5/-10") is True + assert date_in_mars_rule(d(-3), "-1/-5/-10") is False + + def test_list_mixed_formats(self): + rule = "20250125/-5/2023-04-23" + assert date_in_mars_rule(date(2025, 1, 25), rule) is True + assert date_in_mars_rule(d(-5), rule) is True + assert date_in_mars_rule(date(2023, 4, 23), rule) is True + assert date_in_mars_rule(date(2025, 1, 26), rule) is False + + # Range rules + def test_range(self): + assert date_in_mars_rule(d(-1), "-1/to/-20") is True + assert date_in_mars_rule(d(-10), "-1/to/-20") is True + assert date_in_mars_rule(d(-20), "-1/to/-20") is True + assert date_in_mars_rule(d(0), "-1/to/-20") is False + assert date_in_mars_rule(d(-21), "-1/to/-20") is False + # Inverted range covers same dates + assert date_in_mars_rule(d(-10), "-20/to/-1") is True + + # Stepped range rules + def test_stepped_range(self): + # On step: -4, -8, -12, -16, -20 + assert date_in_mars_rule(d(-4), "-4/to/-20/by/4") is True + assert date_in_mars_rule(d(-8), "-4/to/-20/by/4") is True + assert date_in_mars_rule(d(-20), "-4/to/-20/by/4") is True + # Off step or outside range + assert date_in_mars_rule(d(-5), "-4/to/-20/by/4") is False + assert date_in_mars_rule(d(-3), "-4/to/-20/by/4") is False + + def test_stepped_range_absolute(self): + # 2024-02-21/to/2025-03-01/by/10 + start = date(2024, 2, 21) + rule = "2024-02-21/to/2025-03-01/by/10" + assert date_in_mars_rule(start, rule) is True + assert date_in_mars_rule(start + timedelta(days=10), rule) is True + assert date_in_mars_rule(start + timedelta(days=11), rule) is False + + +# --------------------------------------------------------------------------- +# date_check — from the user's examples +# --------------------------------------------------------------------------- + + +class TestDateCheckNewStyle: + def test_single_relative(self): + assert date_check(ds(-1), ["-1"]) is True + with pytest.raises(DateError): + date_check(ds(-2), ["-1"]) + + def test_list_rule(self): + assert date_check(ds(-1), ["-1/-5/-10"]) is True + assert date_check(ds(-5), ["-1/-5/-10"]) is True + with pytest.raises(DateError): + date_check(ds(-3), ["-1/-5/-10"]) + + def test_range_rule(self): + assert date_check(ds(-1), ["-1/to/-20"]) is True + assert date_check(ds(-10), ["-1/to/-20"]) is True + with pytest.raises(DateError): + date_check(ds(-21), ["-1/to/-20"]) + + def test_stepped_range_rule(self): + assert date_check(ds(-4), ["-4/to/-20/by/4"]) is True + assert date_check(ds(-8), ["-4/to/-20/by/4"]) is True + with pytest.raises(DateError): + date_check(ds(-5), ["-4/to/-20/by/4"]) + + def test_mixed_formats(self): + rule = ["20250125/-5/2023-04-23"] + assert date_check("20250125", rule) is True + assert date_check(ds(-5), rule) is True + with pytest.raises(DateError): + date_check("20250126", rule) + + # --- Multiple rules: OR logic --- + + def test_or_logic(self): + # Matches first rule + assert date_check(ds(-1), ["-1/-5/-10", "-20/to/-30"]) is True + # Matches second rule + assert date_check(ds(-25), ["-1/-5/-10", "-20/to/-30"]) is True + # Matches no rule + with pytest.raises(DateError): + date_check(ds(-15), ["-1/-5/-10", "-20/to/-30"]) + + # --- User date as Mars string (range/list) --- + + def test_user_date_range(self): + # All within rule + assert date_check(f"{ds(-5)}/to/{ds(-10)}", ["-1/to/-20"]) is True + # Partially outside rule + with pytest.raises(DateError): + date_check(f"{ds(-1)}/to/{ds(-25)}", ["-1/to/-20"]) + + def test_user_date_list(self): + # All match + assert date_check(f"{ds(-1)}/{ds(-5)}/{ds(-10)}", ["-1/to/-20"]) is True + # One outside + with pytest.raises(DateError): + date_check(f"{ds(-1)}/{ds(-5)}/{ds(-25)}", ["-1/to/-20"]) + + def test_user_date_stepped_range(self): + # Exact match + assert date_check(f"{ds(-4)}/to/{ds(-20)}/by/4", ["-4/to/-20/by/4"]) is True + # Subset of allowed range + assert date_check(f"{ds(-4)}/to/{ds(-20)}/by/4", ["-1/to/-20"]) is True + # Exceeds rule + with pytest.raises(DateError): + date_check(f"{ds(-4)}/to/{ds(-24)}/by/4", ["-1/to/-20"]) + + def test_empty_allowed_values(self): + assert date_check(ds(-1), []) is True + + +# --------------------------------------------------------------------------- +# date_check — old-style rules: backward compatibility +# --------------------------------------------------------------------------- + + +class TestDateCheckOldStyle: + def test_single(self): + assert date_check(ds(-32), [">30d"]) is True + with pytest.raises(DateError): + date_check(ds(-5), [">30d"]) + + def test_multiple_and_logic(self): + # Both conditions must be satisfied + assert date_check(ds(-32), [">30d", "<40d"]) is True + with pytest.raises(DateError): + date_check(ds(-32), [">30d", "<20d"]) + + def test_range(self): + # All dates in range must satisfy rule + assert date_check(f"{ds(-60)}/to/{ds(-40)}", [">30d"]) is True + with pytest.raises(DateError): + date_check(f"{ds(-60)}/to/{ds(-25)}", [">30d"])