From b564be7e82fde950ef3a329abc70c61ffdf5bad4 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Thu, 27 Feb 2025 17:47:30 +0200 Subject: [PATCH 1/3] make path argument optional --- .../declarative_component_schema.yaml | 12 ++- .../models/declarative_component_schema.py | 8 +- .../declarative/requesters/http_requester.py | 77 +++++++++++++++---- .../declarative/requesters/requester.py | 8 +- airbyte_cdk/sources/types.py | 1 + .../requesters/test_http_requester.py | 2 +- 6 files changed, 84 insertions(+), 24 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index c664e237a..a79ea40c0 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -1754,7 +1754,6 @@ definitions: type: object required: - type - - path - url_base properties: type: @@ -1766,9 +1765,18 @@ definitions: type: string interpolation_context: - config + - next_page_token + - stream_interval + - stream_partition + - stream_slice + - creation_response + - polling_response + - download_target examples: - "https://connect.squareup.com/v2" - - "{{ config['base_url'] or 'https://app.posthog.com'}}/api/" + - "{{ config['base_url'] or 'https://app.posthog.com'}}/api" + - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups" + - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}" path: title: URL Path description: Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index abe4d89cf..855550d2c 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -2048,12 +2048,14 @@ class HttpRequester(BaseModel): description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", examples=[ "https://connect.squareup.com/v2", - "{{ config['base_url'] or 'https://app.posthog.com'}}/api/", + "{{ config['base_url'] or 'https://app.posthog.com'}}/api", + "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", + "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", ], title="API Base URL", ) - path: str = Field( - ..., + path: Optional[str] = Field( + None, description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", examples=[ "/products", diff --git a/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte_cdk/sources/declarative/requesters/http_requester.py index 8a7b6aba0..3cf3c8282 100644 --- a/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -25,7 +25,7 @@ from airbyte_cdk.sources.streams.call_rate import APIBudget from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler -from airbyte_cdk.sources.types import Config, StreamSlice, StreamState +from airbyte_cdk.sources.types import Config, EmptyString, StreamSlice, StreamState from airbyte_cdk.utils.mapping_helpers import combine_mappings @@ -49,9 +49,10 @@ class HttpRequester(Requester): name: str url_base: Union[InterpolatedString, str] - path: Union[InterpolatedString, str] config: Config parameters: InitVar[Mapping[str, Any]] + + path: Optional[Union[InterpolatedString, str]] = None authenticator: Optional[DeclarativeAuthenticator] = None http_method: Union[str, HttpMethod] = HttpMethod.GET request_options_provider: Optional[InterpolatedRequestOptionsProvider] = None @@ -66,7 +67,9 @@ class HttpRequester(Requester): def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._url_base = InterpolatedString.create(self.url_base, parameters=parameters) - self._path = InterpolatedString.create(self.path, parameters=parameters) + self._path = InterpolatedString.create( + self.path if self.path else EmptyString, parameters=parameters + ) if self.request_options_provider is None: self._request_options_provider = InterpolatedRequestOptionsProvider( config=self.config, parameters=parameters @@ -112,27 +115,50 @@ def exit_on_rate_limit(self, value: bool) -> None: def get_authenticator(self) -> DeclarativeAuthenticator: return self._authenticator - def get_url_base(self) -> str: - return os.path.join(self._url_base.eval(self.config), "") - - def get_path( + def _get_interpolation_context( self, - *, - stream_state: Optional[StreamState], - stream_slice: Optional[StreamSlice], - next_page_token: Optional[Mapping[str, Any]], - ) -> str: - kwargs = { + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + return { "stream_slice": stream_slice, "next_page_token": next_page_token, - # update the interpolation context with extra fields, if passed. + # update the context with extra fields, if passed. **( stream_slice.extra_fields if stream_slice is not None and hasattr(stream_slice, "extra_fields") else {} ), } - path = str(self._path.eval(self.config, **kwargs)) + + def get_url_base( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: + interpolation_context = self._get_interpolation_context( + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + ) + return os.path.join(self._url_base.eval(self.config, **interpolation_context), EmptyString) + + def get_path( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: + interpolation_context = self._get_interpolation_context( + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + ) + path = str(self._path.eval(self.config, **interpolation_context)) return path.lstrip("/") def get_method(self) -> HttpMethod: @@ -330,7 +356,20 @@ def _request_body_json( @classmethod def _join_url(cls, url_base: str, path: str) -> str: - return urljoin(url_base, path) + """ + Joins a base URL with a given path and returns the resulting URL with any trailing slash removed. + + This method ensures that there are no duplicate slashes when concatenating the base URL and the path, + which is useful when the full URL is provided from an interpolation context. + + Args: + url_base (str): The base URL to which the path will be appended. + path (str): The path to join with the base URL. + + Returns: + str: The concatenated URL with the trailing slash (if any) removed. + """ + return urljoin(url_base, path).rstrip("/") def send_request( self, @@ -347,7 +386,11 @@ def send_request( request, response = self._http_client.send_request( http_method=self.get_method().value, url=self._join_url( - self.get_url_base(), + self.get_url_base( + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + ), path or self.get_path( stream_state=stream_state, diff --git a/airbyte_cdk/sources/declarative/requesters/requester.py b/airbyte_cdk/sources/declarative/requesters/requester.py index 604b2faba..ddda1ddba 100644 --- a/airbyte_cdk/sources/declarative/requesters/requester.py +++ b/airbyte_cdk/sources/declarative/requesters/requester.py @@ -35,7 +35,13 @@ def get_authenticator(self) -> DeclarativeAuthenticator: pass @abstractmethod - def get_url_base(self) -> str: + def get_url_base( + self, + *, + stream_state: Optional[StreamState], + stream_slice: Optional[StreamSlice], + next_page_token: Optional[Mapping[str, Any]], + ) -> str: """ :return: URL base for the API endpoint e.g: if you wanted to hit https://myapi.com/v1/some_entity then this should return "https://myapi.com/v1/" """ diff --git a/airbyte_cdk/sources/types.py b/airbyte_cdk/sources/types.py index d4db76f87..6ee7f652a 100644 --- a/airbyte_cdk/sources/types.py +++ b/airbyte_cdk/sources/types.py @@ -14,6 +14,7 @@ Config = Mapping[str, Any] ConnectionDefinition = Mapping[str, Any] StreamState = Mapping[str, Any] +EmptyString = str() class Record(Mapping[str, Any]): diff --git a/unit_tests/sources/declarative/requesters/test_http_requester.py b/unit_tests/sources/declarative/requesters/test_http_requester.py index dfe78011a..a1229579f 100644 --- a/unit_tests/sources/declarative/requesters/test_http_requester.py +++ b/unit_tests/sources/declarative/requesters/test_http_requester.py @@ -825,7 +825,7 @@ def test_send_request_stream_slice_next_page_token(): "test_trailing_slash_on_path", "https://airbyte.io", "/my_endpoint/", - "https://airbyte.io/my_endpoint/", + "https://airbyte.io/my_endpoint", ), ( "test_nested_path_no_leading_slash", From 748d4b702ed0bcc872722d8446918ebd3dd26863 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 3 Mar 2025 21:21:07 +0200 Subject: [PATCH 2/3] updated after the Maxime's review --- .../declarative/requesters/http_requester.py | 23 +----- .../paginators/default_paginator.py | 29 ++++++- .../requesters/paginators/no_pagination.py | 7 +- .../requesters/paginators/paginator.py | 9 ++- .../retrievers/simple_retriever.py | 25 +++++- airbyte_cdk/utils/mapping_helpers.py | 19 ++++- .../paginators/test_default_paginator.py | 78 ++++++++++++++++++- 7 files changed, 156 insertions(+), 34 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte_cdk/sources/declarative/requesters/http_requester.py index 3cf3c8282..8a64fae60 100644 --- a/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -26,7 +26,7 @@ from airbyte_cdk.sources.streams.http import HttpClient from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler from airbyte_cdk.sources.types import Config, EmptyString, StreamSlice, StreamState -from airbyte_cdk.utils.mapping_helpers import combine_mappings +from airbyte_cdk.utils.mapping_helpers import combine_mappings, get_interpolation_context @dataclass @@ -115,23 +115,6 @@ def exit_on_rate_limit(self, value: bool) -> None: def get_authenticator(self) -> DeclarativeAuthenticator: return self._authenticator - def _get_interpolation_context( - self, - stream_state: Optional[StreamState] = None, - stream_slice: Optional[StreamSlice] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: - return { - "stream_slice": stream_slice, - "next_page_token": next_page_token, - # update the context with extra fields, if passed. - **( - stream_slice.extra_fields - if stream_slice is not None and hasattr(stream_slice, "extra_fields") - else {} - ), - } - def get_url_base( self, *, @@ -139,7 +122,7 @@ def get_url_base( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> str: - interpolation_context = self._get_interpolation_context( + interpolation_context = get_interpolation_context( stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token, @@ -153,7 +136,7 @@ def get_path( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> str: - interpolation_context = self._get_interpolation_context( + interpolation_context = get_interpolation_context( stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token, diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py b/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py index bd640ad19..ca2405b44 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py @@ -25,6 +25,7 @@ from airbyte_cdk.sources.types import Config, Record, StreamSlice, StreamState from airbyte_cdk.utils.mapping_helpers import ( _validate_component_request_option_paths, + get_interpolation_context, ) @@ -150,11 +151,22 @@ def next_page_token( else: return None - def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]: + def path( + self, + next_page_token: Optional[Mapping[str, Any]], + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[StreamSlice] = None, + ) -> Optional[str]: token = next_page_token.get("next_page_token") if next_page_token else None if token and self.page_token_option and isinstance(self.page_token_option, RequestPath): + # make additional interpolation context + interpolation_context = get_interpolation_context( + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + ) # Replace url base to only return the path - return str(token).replace(self.url_base.eval(self.config), "") # type: ignore # url_base is casted to a InterpolatedString in __post_init__ + return str(token).replace(self.url_base.eval(self.config, **interpolation_context), "") # type: ignore # url_base is casted to a InterpolatedString in __post_init__ else: return None @@ -258,8 +270,17 @@ def next_page_token( response, last_page_size, last_record, last_page_token_value ) - def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]: - return self._decorated.path(next_page_token) + def path( + self, + next_page_token: Optional[Mapping[str, Any]], + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[StreamSlice] = None, + ) -> Optional[str]: + return self._decorated.path( + next_page_token=next_page_token, + stream_state=stream_state, + stream_slice=stream_slice, + ) def get_request_params( self, diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py b/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py index 7de91f5e9..b3b1d3b66 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py @@ -19,7 +19,12 @@ class NoPagination(Paginator): parameters: InitVar[Mapping[str, Any]] - def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]: + def path( + self, + next_page_token: Optional[Mapping[str, Any]], + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[StreamSlice] = None, + ) -> Optional[str]: return None def get_request_params( diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py b/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py index 8b1fea69b..f8c31d4f5 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py @@ -11,7 +11,7 @@ from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import ( RequestOptionsProvider, ) -from airbyte_cdk.sources.types import Record +from airbyte_cdk.sources.types import Record, StreamSlice @dataclass @@ -49,7 +49,12 @@ def next_page_token( pass @abstractmethod - def path(self, next_page_token: Optional[Mapping[str, Any]]) -> Optional[str]: + def path( + self, + next_page_token: Optional[Mapping[str, Any]], + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[StreamSlice] = None, + ) -> Optional[str]: """ Returns the URL path to hit to fetch the next page of records diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index a535a9b3d..df41e14a7 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -234,13 +234,22 @@ def _request_body_json( raise ValueError("Request body json cannot be a string") return body_json - def _paginator_path(self, next_page_token: Optional[Mapping[str, Any]] = None) -> Optional[str]: + def _paginator_path( + self, + next_page_token: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[StreamSlice] = None, + ) -> Optional[str]: """ If the paginator points to a path, follow it, else return nothing so the requester is used. :param next_page_token: :return: """ - return self._paginator.path(next_page_token=next_page_token) + return self._paginator.path( + next_page_token=next_page_token, + stream_state=stream_state, + stream_slice=stream_slice, + ) def _parse_response( self, @@ -299,7 +308,11 @@ def _fetch_next_page( next_page_token: Optional[Mapping[str, Any]] = None, ) -> Optional[requests.Response]: return self.requester.send_request( - path=self._paginator_path(next_page_token=next_page_token), + path=self._paginator_path( + next_page_token=next_page_token, + stream_state=stream_state, + stream_slice=stream_slice, + ), stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token, @@ -570,7 +583,11 @@ def _fetch_next_page( next_page_token: Optional[Mapping[str, Any]] = None, ) -> Optional[requests.Response]: return self.requester.send_request( - path=self._paginator_path(next_page_token=next_page_token), + path=self._paginator_path( + next_page_token=next_page_token, + stream_state=stream_state, + stream_slice=stream_slice, + ), stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token, diff --git a/airbyte_cdk/utils/mapping_helpers.py b/airbyte_cdk/utils/mapping_helpers.py index bce3a849b..bfe2b7709 100644 --- a/airbyte_cdk/utils/mapping_helpers.py +++ b/airbyte_cdk/utils/mapping_helpers.py @@ -10,7 +10,7 @@ RequestOption, RequestOptionType, ) -from airbyte_cdk.sources.types import Config +from airbyte_cdk.sources.types import Config, StreamSlice, StreamState def _merge_mappings( @@ -143,3 +143,20 @@ def _validate_component_request_option_paths( ) except ValueError as error: raise ValueError(error) + + +def get_interpolation_context( + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, +) -> Mapping[str, Any]: + return { + "stream_slice": stream_slice, + "next_page_token": next_page_token, + # update the context with extra fields, if passed. + **( + stream_slice.extra_fields + if stream_slice is not None and hasattr(stream_slice, "extra_fields") + else {} + ), + } diff --git a/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py b/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py index 944a0eda9..6e7b60a92 100644 --- a/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +++ b/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py @@ -3,7 +3,7 @@ # import json -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock import pytest import requests @@ -26,7 +26,7 @@ PageIncrement, ) from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath -from airbyte_cdk.sources.declarative.types import Record +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState @pytest.mark.parametrize( @@ -473,3 +473,77 @@ def test_request_option_mapping_validator(): parameters={}, ), ) + + +def test_path_returns_none_when_no_token() -> None: + page_token_option = RequestPath(parameters={}) + paginator = DefaultPaginator( + pagination_strategy=Mock(), + config={}, + url_base="https://domain.com", + parameters={}, + page_token_option=page_token_option, + ) + result = paginator.path(None) + + assert result is None + + +def test_path_returns_none_when_option_not_request_path() -> None: + token_value = "https://domain.com/next_url" + next_page_token = {"next_page_token": token_value} + + # Use a RequestOption instead of RequestPath. + page_token_option = RequestOption( + inject_into=RequestOptionType.request_parameter, + field_name="some_field", + parameters={}, + ) + paginator = DefaultPaginator( + pagination_strategy=Mock(), + config={}, + url_base="https://domain.com", + parameters={}, + page_token_option=page_token_option, + ) + result = paginator.path(next_page_token) + assert result is None + + +def test_path_with_additional_interpolation_context() -> None: + page_token_option = RequestPath(parameters={}) + paginator = DefaultPaginator( + pagination_strategy=Mock(), + config={}, + url_base="https://api.domain.com/{{ stream_slice['campaign_id'] }}", + parameters={}, + page_token_option=page_token_option, + ) + # define stream_state here + stream_state = {"state": "state_value"} + # define stream_slice here + stream_slice = StreamSlice( + partition={ + "campaign_id": "123_abcd", + }, + cursor_slice={ + "start": "A", + "end": "B", + }, + extra_fields={ + "extra_field_A": "value_A", + "extra_field_B": "value_B", + }, + ) + # define next_page_token here + next_page_token = { + "next_page_token": "https://api.domain.com/123_abcd/some_next_page_token_here" + } + + expected_after_interpolation = "/some_next_page_token_here" + + assert expected_after_interpolation == paginator.path( + next_page_token=next_page_token, + stream_state=stream_state, + stream_slice=stream_slice, + ) From a4fb3eb027d9209d5c3974c359f90ef190234dce Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Mon, 3 Mar 2025 21:22:19 +0200 Subject: [PATCH 3/3] formatted --- .../declarative/models/declarative_component_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 6df2e3ff5..a49b66c03 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -939,7 +939,7 @@ class MinMaxDatetime(BaseModel): ) datetime_format: Optional[str] = Field( "", - description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', + description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`, `001`, ..., `999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], title="Datetime Format", ) @@ -1545,7 +1545,7 @@ class DatetimeBasedCursor(BaseModel): ) datetime_format: str = Field( ..., - description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", + description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%_ms**: Millisecond (zero-padded to 3 digits) - `000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], title="Outgoing Datetime Format", )