50395 2 2

.github/workflows/autofix-command.yml

@@ -75,7 +75,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.8.4"
+          poetry-version: "2.0.1"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

.github/workflows/connector-tests.yml

@@ -128,7 +128,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       # Create initial pending status for test report
       - name: Create Pending Test Report Status
         if: steps.no_changes.outputs.status != 'cancelled'

.github/workflows/pdoc_preview.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.8.4"
+          poetry-version: "2.0.1"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

.github/workflows/pdoc_publish.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.8.4"
+          poetry-version: "2.0.1"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

.github/workflows/poetry-lock-command.yml

@@ -36,6 +36,17 @@ jobs:
         with:
           pr: ${{ github.event.inputs.pr }}
 
+      - name: Check for blank or missing poetry.lock
+        run: |
+          if [ ! -s "poetry.lock" ]; then
+            echo "poetry.lock missing or blank. Fetching from main branch..."
+            git fetch origin main
+            git checkout origin/main -- poetry.lock
+            echo "Lock file restored from main."
+          else
+            echo "poetry.lock found. Proceeding."
+          fi
+
       - name: Get PR info
         id: pr-info
         run: |
@@ -72,7 +83,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.8.4"
+          poetry-version: "2.0.1"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

.github/workflows/pytest_fast.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.8.4"
+          poetry-version: "2.0.1"
 
       - name: Check Poetry lock file is current
         run: poetry check
@@ -44,7 +44,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.8.4"
+          poetry-version: "2.0.1"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

.github/workflows/pytest_matrix.yml

@@ -30,7 +30,7 @@ jobs:
         python-version: [
             "3.10",
             "3.11",
-            #'3.12', # Currently blocked by Pendulum
+            # "3.12",  # `requests-cache` blocker: https://github.com/airbytehq/airbyte-python-cdk/issues/299
           ]
         os: [
             Ubuntu,
@@ -61,7 +61,7 @@ jobs:
         uses: Gr1N/setup-poetry@v9
         if: steps.changes.outputs.src == 'true'
         with:
-          poetry-version: "1.8.4"
+          poetry-version: "2.0.1"
       - name: Set up Python
         uses: actions/setup-python@v5
         if: steps.changes.outputs.src == 'true'

.github/workflows/python_lint.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.8.4"
+          poetry-version: "2.0.1"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

.github/workflows/test-command.yml

@@ -85,7 +85,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.8.4"
+          poetry-version: "2.0.1"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

.pre-commit-config.yaml

@@ -0,0 +1,63 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+exclude: |
+  (?x)(
+    # Python/system files
+    ^.*/__init__\.py$|
+    ^.*?/\.venv/.*$|
+    ^.*?/node_modules/.*$|
+
+    # Generated/test files
+    ^.*?/\.pytest_cache/.*$|
+    ^.*?/__pycache__/.*$|
+    ^.*?/\.mypy_cache/.*$|
+    ^.*?/\.ruff_cache/.*$
+
+    # Package management
+    ^.*?/poetry\.lock$|
+    ^.*?/package-lock\.json$|
+    ^.*?/pnpm-lock\.yaml$|
+
+    # Build and test artifacts
+    ^.*?/build/.*$|
+    ^.*?/dist/.*$|
+    ^.*?/\.coverage$|
+    ^.*?/coverage\.xml$|
+  )
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+      - id: check-toml
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.3
+    hooks:
+      # Run the linter with repo-defined settings
+      - id: ruff
+        args: [--fix]
+
+      # Run the formatter with repo-defined settings
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.0.3
+    hooks:
+      - id: prettier
+        types_or: [json, yaml]
+        additional_dependencies:
+          - [email protected]
+
+  - repo: local
+    hooks:
+      - id: addlicense
+        name: Add license headers
+        entry: addlicense -c "Airbyte, Inc." -l apache -v -f LICENSE_SHORT
+        language: golang
+        additional_dependencies: [github.com/google/[email protected]]
+        files: \.py$

LICENSE_SHORT

@@ -0,0 +1 @@
+Copyright (c) 2025 Airbyte, Inc., all rights reserved.

airbyte_cdk/__init__.py

@@ -92,7 +92,7 @@
 from .sources.declarative.declarative_stream import DeclarativeStream
 from .sources.declarative.decoders import Decoder, JsonDecoder
 from .sources.declarative.exceptions import ReadException
-from .sources.declarative.extractors import DpathExtractor, RecordSelector
+from .sources.declarative.extractors import DpathExtractor, DpathEnhancingExtractor, RecordSelector
 from .sources.declarative.extractors.record_extractor import RecordExtractor
 from .sources.declarative.extractors.record_filter import RecordFilter
 from .sources.declarative.incremental import DatetimeBasedCursor
@@ -234,6 +234,7 @@
     "DefaultPaginator",
     "DefaultRequestOptionsProvider",
     "DpathExtractor",
+    "DpathEnhancingExtractor",
     "FieldPointer",
     "HttpMethod",
     "HttpRequester",

airbyte_cdk/cli/source_declarative_manifest/_run.py

@@ -21,7 +21,6 @@
 import sys
 import traceback
 from collections.abc import Mapping
-from datetime import datetime
 from pathlib import Path
 from typing import Any, cast
 
@@ -44,6 +43,7 @@
 )
 from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
 from airbyte_cdk.sources.source import TState
+from airbyte_cdk.utils.datetime_helpers import ab_datetime_now
 
 
 class SourceLocalYaml(YamlDeclarativeSource):
@@ -101,7 +101,7 @@ def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
                         type=Type.TRACE,
                         trace=AirbyteTraceMessage(
                             type=TraceType.ERROR,
-                            emitted_at=int(datetime.now().timestamp() * 1000),
+                            emitted_at=ab_datetime_now().to_epoch_millis(),
                             error=AirbyteErrorTraceMessage(
                                 message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
                                 stack_trace=traceback.format_exc(),
@@ -191,7 +191,7 @@ def create_declarative_source(
                         type=Type.TRACE,
                         trace=AirbyteTraceMessage(
                             type=TraceType.ERROR,
-                            emitted_at=int(datetime.now().timestamp() * 1000),
+                            emitted_at=ab_datetime_now().to_epoch_millis(),
                             error=AirbyteErrorTraceMessage(
                                 message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
                                 stack_trace=traceback.format_exc(),

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -3,7 +3,6 @@
 #
 
 import dataclasses
-from datetime import datetime
 from typing import Any, List, Mapping
 
 from airbyte_cdk.connector_builder.message_grouper import MessageGrouper
@@ -21,6 +20,7 @@
     ModelToComponentFactory,
 )
 from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
+from airbyte_cdk.utils.datetime_helpers import ab_datetime_now
 from airbyte_cdk.utils.traced_exception import AirbyteTracedException
 
 DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE = 5
@@ -114,4 +114,4 @@ def resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage:
 
 
 def _emitted_at() -> int:
-    return int(datetime.now().timestamp()) * 1000
+    return ab_datetime_now().to_epoch_millis()

airbyte_cdk/sources/declarative/async_job/job_orchestrator.py

@@ -437,10 +437,10 @@ def create_and_get_completed_partitions(self) -> Iterable[AsyncPartition]:
                 yield from self._process_running_partitions_and_yield_completed_ones()
                 self._wait_on_status_update()
             except Exception as exception:
+                LOGGER.warning(
+                    f"Caught exception that stops the processing of the jobs: {exception}. Traceback: {traceback.format_exc()}"
+                )
                 if self._is_breaking_exception(exception):
-                    LOGGER.warning(
-                        f"Caught exception that stops the processing of the jobs: {exception}"
-                    )
                     self._abort_all_running_jobs()
                     raise exception
 
@@ -482,16 +482,16 @@ def _is_breaking_exception(self, exception: Exception) -> bool:
             and exception.failure_type == FailureType.config_error
         )
 
-    def fetch_records(self, partition: AsyncPartition) -> Iterable[Mapping[str, Any]]:
+    def fetch_records(self, async_jobs: Iterable[AsyncJob]) -> Iterable[Mapping[str, Any]]:
         """
-        Fetches records from the given partition's jobs.
+        Fetches records from the given jobs.
 
         Args:
-            partition (AsyncPartition): The partition containing the jobs.
+            async_jobs Iterable[AsyncJob]: The list of AsyncJobs.
 
         Yields:
             Iterable[Mapping[str, Any]]: The fetched records from the jobs.
         """
-        for job in partition.jobs:
+        for job in async_jobs:
             yield from self._job_repository.fetch_records(job)
             self._job_repository.delete(job)

airbyte_cdk/sources/declarative/auth/jwt.py

@@ -3,6 +3,7 @@
 #
 
 import base64
+import json
 from dataclasses import InitVar, dataclass
 from datetime import datetime
 from typing import Any, Mapping, Optional, Union
@@ -104,21 +105,21 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         )
 
     def _get_jwt_headers(self) -> dict[str, Any]:
-        """ "
+        """
         Builds and returns the headers used when signing the JWT.
         """
-        headers = self._additional_jwt_headers.eval(self.config)
+        headers = self._additional_jwt_headers.eval(self.config, json_loads=json.loads)
         if any(prop in headers for prop in ["kid", "alg", "typ", "cty"]):
             raise ValueError(
                 "'kid', 'alg', 'typ', 'cty' are reserved headers and should not be set as part of 'additional_jwt_headers'"
             )
 
         if self._kid:
-            headers["kid"] = self._kid.eval(self.config)
+            headers["kid"] = self._kid.eval(self.config, json_loads=json.loads)
         if self._typ:
-            headers["typ"] = self._typ.eval(self.config)
+            headers["typ"] = self._typ.eval(self.config, json_loads=json.loads)
         if self._cty:
-            headers["cty"] = self._cty.eval(self.config)
+            headers["cty"] = self._cty.eval(self.config, json_loads=json.loads)
         headers["alg"] = self._algorithm
         return headers
 
@@ -130,18 +131,19 @@ def _get_jwt_payload(self) -> dict[str, Any]:
         exp = now + self._token_duration if isinstance(self._token_duration, int) else now
         nbf = now
 
-        payload = self._additional_jwt_payload.eval(self.config)
+        payload = self._additional_jwt_payload.eval(self.config, json_loads=json.loads)
         if any(prop in payload for prop in ["iss", "sub", "aud", "iat", "exp", "nbf"]):
             raise ValueError(
                 "'iss', 'sub', 'aud', 'iat', 'exp', 'nbf' are reserved properties and should not be set as part of 'additional_jwt_payload'"
             )
 
         if self._iss:
-            payload["iss"] = self._iss.eval(self.config)
+            payload["iss"] = self._iss.eval(self.config, json_loads=json.loads)
         if self._sub:
-            payload["sub"] = self._sub.eval(self.config)
+            payload["sub"] = self._sub.eval(self.config, json_loads=json.loads)
         if self._aud:
-            payload["aud"] = self._aud.eval(self.config)
+            payload["aud"] = self._aud.eval(self.config, json_loads=json.loads)
+
         payload["iat"] = now
         payload["exp"] = exp
         payload["nbf"] = nbf
@@ -151,7 +153,7 @@ def _get_secret_key(self) -> str:
         """
         Returns the secret key used to sign the JWT.
         """
-        secret_key: str = self._secret_key.eval(self.config)
+        secret_key: str = self._secret_key.eval(self.config, json_loads=json.loads)
         return (
             base64.b64encode(secret_key.encode()).decode()
             if self._base64_encode_secret_key
@@ -176,7 +178,11 @@ def _get_header_prefix(self) -> Union[str, None]:
         """
         Returns the header prefix to be used when attaching the token to the request.
         """
-        return self._header_prefix.eval(self.config) if self._header_prefix else None
+        return (
+            self._header_prefix.eval(self.config, json_loads=json.loads)
+            if self._header_prefix
+            else None
+        )
 
     @property
     def auth_header(self) -> str: