-
Notifications
You must be signed in to change notification settings - Fork 4
feat(datasource): mars-style date matching #168
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
peshence
wants to merge
1
commit into
main
Choose a base branch
from
feat/mars-style-date-matching
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,20 +3,57 @@ | |
|
|
||
| from dateutil.relativedelta import relativedelta | ||
|
|
||
| from ..exceptions import ServerError | ||
|
|
||
|
|
||
| class DateError(Exception): | ||
| """Custom exception for date-related errors.""" | ||
|
|
||
| pass | ||
|
|
||
|
|
||
| def date_check(date: str, allowed_values: list[str]): | ||
| """ | ||
| Process special match rules for DATE constraints. | ||
|
|
||
| :param date: Date to check, can be a single, list or range of dates (Mars date format) | ||
| :param allowed_values: List of rules. All rules must be the same style: | ||
|
|
||
| Old style (e.g. ">30d", "<40d"): | ||
| Each date must satisfy ALL rules (AND logic). | ||
|
|
||
| New style – Mars date strings (e.g. "-1/-5/-10", "-1/to/-20", "-4/to/-20/by/4"): | ||
| Each individual date must match AT LEAST ONE rule (OR logic). | ||
| """ | ||
| if not isinstance(allowed_values, list): | ||
| raise ServerError("Allowed values must be a list") | ||
|
|
||
| if not allowed_values: | ||
| return True | ||
|
|
||
| if all(map(is_old_style_rule, allowed_values)): | ||
| # Old-style: every rule must pass | ||
| for rule in allowed_values: | ||
| if not date_check_single_rule(date, rule): | ||
| return False | ||
| return True | ||
|
|
||
| # New-style Mars date rules: each user date must match at least one rule | ||
| user_dates = expand_mars_dates(date) | ||
| for user_date in user_dates: | ||
| if not any(date_in_mars_rule(user_date, rule) for rule in allowed_values): | ||
| raise DateError(f"Date {user_date} does not match any allowed date rule: {allowed_values}") | ||
|
|
||
peshence marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return True | ||
|
|
||
|
|
||
| def check_single_date(date, offset, offset_fmted, after=False): | ||
| # Date is relative (0 = now, -1 = one day ago) | ||
| if str(date)[0] == "0" or str(date)[0] == "-": | ||
| date_offset = int(date) | ||
| dt = datetime.today() + timedelta(days=date_offset) | ||
|
|
||
| if after and dt >= offset: | ||
| if after and dt > offset: | ||
| raise DateError("Date is too recent, expected < {}".format(offset_fmted)) | ||
| elif not after and dt < offset: | ||
| raise DateError("Date is too old, expected > {}".format(offset_fmted)) | ||
|
|
@@ -28,7 +65,7 @@ def check_single_date(date, offset, offset_fmted, after=False): | |
| dt = datetime.strptime(date, "%Y%m%d") | ||
| except ValueError: | ||
| raise DateError("Invalid date, expected real date in YYYYMMDD format") | ||
| if after and dt >= offset: | ||
| if after and dt > offset: | ||
| raise DateError("Date is too recent, expected < {}".format(offset_fmted)) | ||
peshence marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| elif not after and dt < offset: | ||
| raise DateError("Date is too old, expected > {}".format(offset_fmted)) | ||
|
|
@@ -52,26 +89,125 @@ def parse_relativedelta(time_str): | |
| return relativedelta(days=time_dict["d"], hours=time_dict["h"], minutes=time_dict["m"]) | ||
|
|
||
|
|
||
| def date_check(date, allowed_values: list): | ||
| def is_old_style_rule(rule: str) -> bool: | ||
| """Returns True if rule is old-style (starts with > or <).""" | ||
| return rule.strip()[0] in (">", "<") | ||
peshence marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
|
|
||
| def parse_mars_date_token(token: str) -> datetime: | ||
| """Parse a single Mars date token to a datetime. | ||
|
|
||
| Supports: | ||
| - Relative dates: 0 (today), -1 (yesterday), -10, etc. | ||
| - Absolute YYYYMMDD: 20250125 | ||
| - Absolute YYYY-MM-DD: 2023-04-23 | ||
| """ | ||
| Process special match rules for DATE constraints | ||
| token = token.strip() | ||
| if not token: | ||
| raise DateError("Empty date token") | ||
| # Relative date: starts with '-' or '0' (matches existing check_single_date convention) | ||
| if token[0] == "-" or token[0] == "0": | ||
| try: | ||
| offset = int(token) | ||
| return datetime.today().replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=offset) | ||
| except ValueError: | ||
| pass | ||
| # ISO format YYYY-MM-DD | ||
| try: | ||
| return datetime.strptime(token, "%Y-%m-%d") | ||
| except ValueError: | ||
| pass | ||
| # YYYYMMDD format | ||
| try: | ||
| return datetime.strptime(token, "%Y%m%d") | ||
| except ValueError: | ||
| raise DateError(f"Invalid Mars date token: {token!r}") | ||
|
|
||
| :param date: Date to check, can be a string or list of strings | ||
| :param allowed_values: List of allowed values for the date in the format >1d, <2d, >1m, <2h, r"(\\d+)([dhm])". | ||
|
|
||
| def expand_mars_dates(date_str: str) -> list: | ||
| """Expand a Mars date string to a list of date objects. | ||
|
|
||
| Handles single dates, lists, ranges, and ranges with step. | ||
|
|
||
| Examples: | ||
| "-1" -> [date(-1)] | ||
| "-1/-5/-10" -> [date(-1), date(-5), date(-10)] | ||
| "-1/to/-20" -> [date(-1), date(-2), ..., date(-20)] | ||
| "-4/to/-20/by/4" -> [date(-4), date(-8), date(-12), date(-16), date(-20)] | ||
| "20250125/-5/2023-04-23" -> [date(20250125), date(-5), date(2023-04-23)] | ||
| "2024-02-21/to/2025-03-01/by/10" -> dates every 10 days across the range | ||
| """ | ||
| if not isinstance(allowed_values, list): | ||
| raise DateError("Allowed values must be a list") | ||
| parts = date_str.split("/") | ||
|
|
||
| # Range syntax: second element is 'to' | ||
| if len(parts) >= 3 and parts[1].strip().lower() == "to": | ||
| start_d = parse_mars_date_token(parts[0]).date() | ||
| end_d = parse_mars_date_token(parts[2]).date() | ||
| step = 1 | ||
| if len(parts) == 5: | ||
| if parts[3].strip().lower() != "by": | ||
| raise DateError(f"Invalid Mars date string: {date_str!r}") | ||
| step = abs(int(parts[4].strip())) | ||
| elif len(parts) != 3: | ||
| raise DateError(f"Invalid Mars date string: {date_str!r}") | ||
|
|
||
| dates = [] | ||
| if start_d <= end_d: | ||
| current = start_d | ||
| while current <= end_d: | ||
| dates.append(current) | ||
| current += timedelta(days=step) | ||
| else: | ||
| current = start_d | ||
| while current >= end_d: | ||
| dates.append(current) | ||
| current -= timedelta(days=step) | ||
| return dates | ||
|
Comment on lines
+147
to
+165
|
||
|
|
||
| # List or single date | ||
| return [parse_mars_date_token(p).date() for p in parts] | ||
|
|
||
|
|
||
| for allowed in allowed_values: | ||
| if not date_check_single_rule(date, allowed): | ||
| def date_in_mars_rule(date_d, rule_str: str) -> bool: | ||
| """Check whether a single date (a date object) is covered by a Mars date rule string. | ||
|
|
||
| The rule string follows the same Mars date syntax: | ||
| - Single: '-1', '20250125', '2023-04-23' | ||
| - List: '-1/-5/-10', '20250125/-5/2023-04-23' | ||
| - Range: '-1/to/-20', '2024-02-21/to/2025-03-01' | ||
| - Range+step: '-4/to/-20/by/4', '2024-02-21/to/2025-03-01/by/10' | ||
| """ | ||
| parts = rule_str.split("/") | ||
|
|
||
| # Range syntax | ||
| if len(parts) >= 3 and parts[1].strip().lower() == "to": | ||
| start_d = parse_mars_date_token(parts[0]).date() | ||
| end_d = parse_mars_date_token(parts[2]).date() | ||
| step = 1 | ||
| if len(parts) == 5: | ||
| if parts[3].strip().lower() != "by": | ||
| raise DateError(f"Invalid Mars date rule: {rule_str!r}") | ||
| step = abs(int(parts[4].strip())) | ||
| elif len(parts) != 3: | ||
| raise DateError(f"Invalid Mars date rule: {rule_str!r}") | ||
|
|
||
| min_d = min(start_d, end_d) | ||
| max_d = max(start_d, end_d) | ||
| if not (min_d <= date_d <= max_d): | ||
| return False | ||
| # Date must fall on a step boundary from start | ||
| return abs((date_d - start_d).days) % step == 0 | ||
|
Comment on lines
+186
to
+199
|
||
|
|
||
| return True | ||
| # List or single: date must equal one of the listed tokens | ||
| for part in parts: | ||
| if parse_mars_date_token(part).date() == date_d: | ||
| return True | ||
| return False | ||
|
|
||
|
|
||
| def date_check_single_rule(date, allowed_values: str): | ||
| """ | ||
| Process special match rules for DATE constraints | ||
| Process special match rules for DATE constraints (old-style rules only). | ||
|
|
||
| :param date: Date to check, can be a string or list of strings | ||
| :param allowed_values: Allowed values for the date in the format >1d, <2d, >1m, <2h, r"(\\d+)([dhm])". | ||
|
|
@@ -89,7 +225,7 @@ def date_check_single_rule(date, allowed_values: str): | |
| elif comp == ">": | ||
| after = True | ||
| else: | ||
| raise DateError(f"Invalid date comparison {comp}, expected < or >") | ||
| raise ServerError(f"Invalid date comparison {comp}, expected < or >") | ||
| now = datetime.today() | ||
| offset = now - parse_relativedelta(offset) | ||
| offset_fmted = offset.strftime("%Y%m%d") | ||
|
|
||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
New-style matching expands the user-provided
datestring viaexpand_mars_dates(date)(including full daily expansion of ranges). A request like19000101/to/21000101can allocate tens of thousands ofdateobjects per datasource match, and much larger ranges can become a DoS vector. Consider avoiding full expansion (e.g., validate user range against rules using boundary/step arithmetic), or enforce a hard maximum number of expanded dates and fail fast withDateErrorif exceeded.