From 0764b630c370d3b579ed6bace30b8f44b44ed872 Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 2 Jul 2025 17:19:30 -0700 Subject: [PATCH 1/8] set token_expiry_is_time_of_expiration in DeclarativeSingleUseRefreshTokenOauth2Authenticator constructor --- .../sources/declarative/parsers/model_to_component_factory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 2f7619ba5..110434c87 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -2801,6 +2801,7 @@ def create_oauth_authenticator( ).eval(config), scopes=model.scopes, token_expiry_date_format=model.token_expiry_date_format, + token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), message_repository=self._message_repository, refresh_token_error_status_codes=model.refresh_token_updater.refresh_token_error_status_codes, refresh_token_error_key=model.refresh_token_updater.refresh_token_error_key, From 4e6b9f230bd129f8271cd2b2eb820064902d5b0c Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 2 Jul 2025 17:19:58 -0700 Subject: [PATCH 2/8] mask access token when logging OAuth token refresh response --- .../streams/http/requests_native_auth/abstract_oauth.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py index 108055f1d..1f22fab49 100644 --- a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +++ b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py @@ -217,10 +217,15 @@ def _make_handled_request(self) -> Any: data=self.build_refresh_request_body(), headers=self.build_refresh_request_headers(), ) + response_json = response.json() + # extract the access token and add to secrets to avoid logging the raw value + access_key = self._extract_access_token(response_json) + if access_key: + add_to_secrets(access_key) # log the response even if the request failed for troubleshooting purposes self._log_response(response) response.raise_for_status() - return response.json() + return response_json except requests.exceptions.RequestException as e: if e.response is not None: if e.response.status_code == 429 or e.response.status_code >= 500: From b1d5d051d0ffa6f1bb8d39c1a10203059e836c3b Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 3 Jul 2025 10:05:46 -0700 Subject: [PATCH 3/8] properly handle case where no access key can be found in response --- .../requests_native_auth/abstract_oauth.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py index 1f22fab49..9a49ae65a 100644 --- a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +++ b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py @@ -218,10 +218,14 @@ def _make_handled_request(self) -> Any: headers=self.build_refresh_request_headers(), ) response_json = response.json() - # extract the access token and add to secrets to avoid logging the raw value - access_key = self._extract_access_token(response_json) - if access_key: - add_to_secrets(access_key) + try: + # extract the access token and add to secrets to avoid logging the raw value + access_key = self._extract_access_token(response_json) + if access_key: + add_to_secrets(access_key) + except ResponseKeysMaxRecurtionReached as e: + ## Could not find the access token in the response, so do nothing + pass # log the response even if the request failed for troubleshooting purposes self._log_response(response) response.raise_for_status() @@ -245,9 +249,7 @@ def _ensure_access_token_in_response(self, response_data: Mapping[str, Any]) -> This method attempts to extract the access token from the provided response data. If the access token is not found, it raises an exception indicating that the token - refresh API response was missing the access token. If the access token is found, - it adds the token to the list of secrets to ensure it is replaced before logging - the response. + refresh API response was missing the access token. Args: response_data (Mapping[str, Any]): The response data from which to extract the access token. @@ -262,9 +264,6 @@ def _ensure_access_token_in_response(self, response_data: Mapping[str, Any]) -> raise Exception( f"Token refresh API response was missing access token {self.get_access_token_name()}" ) - # Add the access token to the list of secrets so it is replaced before logging the response - # An argument could be made to remove the prevous access key from the list of secrets, but unmasking values seems like a security incident waiting to happen... - add_to_secrets(access_key) except ResponseKeysMaxRecurtionReached as e: raise e From bf1eaf2ff23a358fdd405ad5a319139a8068dbac Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 3 Jul 2025 10:34:42 -0700 Subject: [PATCH 4/8] properly handle json exception --- .../requests_native_auth/abstract_oauth.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py index 9a49ae65a..7e5c5b49a 100644 --- a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +++ b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py @@ -217,18 +217,30 @@ def _make_handled_request(self) -> Any: data=self.build_refresh_request_body(), headers=self.build_refresh_request_headers(), ) - response_json = response.json() + + try: + response_json = response.json() + except Exception as e: + # if the json could not be parsed, save the exception to raise it later + json_exception = e + try: - # extract the access token and add to secrets to avoid logging the raw value - access_key = self._extract_access_token(response_json) - if access_key: - add_to_secrets(access_key) + if response_json: + # extract the access token and add to secrets to avoid logging the raw value + access_key = self._extract_access_token(response_json) + if access_key: + add_to_secrets(access_key) except ResponseKeysMaxRecurtionReached as e: - ## Could not find the access token in the response, so do nothing + ## could not find the access token in the response, so do nothing pass + # log the response even if the request failed for troubleshooting purposes self._log_response(response) response.raise_for_status() + + if json_exception: + raise json_exception + return response_json except requests.exceptions.RequestException as e: if e.response is not None: From 10a27b1b4323f793447e278bc292448e1bc81836 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 3 Jul 2025 11:01:59 -0700 Subject: [PATCH 5/8] initialize response_json and json_exception to None at start --- .../streams/http/requests_native_auth/abstract_oauth.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py index 7e5c5b49a..9d77360fe 100644 --- a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +++ b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py @@ -218,6 +218,8 @@ def _make_handled_request(self) -> Any: headers=self.build_refresh_request_headers(), ) + response_json = None + json_exception = None try: response_json = response.json() except Exception as e: @@ -231,7 +233,7 @@ def _make_handled_request(self) -> Any: if access_key: add_to_secrets(access_key) except ResponseKeysMaxRecurtionReached as e: - ## could not find the access token in the response, so do nothing + # could not find the access token in the response, so do nothing pass # log the response even if the request failed for troubleshooting purposes @@ -240,7 +242,7 @@ def _make_handled_request(self) -> Any: if json_exception: raise json_exception - + return response_json except requests.exceptions.RequestException as e: if e.response is not None: From ab062f8e1426f23cbf3ae7d1ff6845b659544038 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 3 Jul 2025 13:43:27 -0700 Subject: [PATCH 6/8] refactor _make_handled_request to be simpler --- .../requests_native_auth/abstract_oauth.py | 35 +++++++------------ 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py index 9d77360fe..710fdf82d 100644 --- a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +++ b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py @@ -218,30 +218,19 @@ def _make_handled_request(self) -> Any: headers=self.build_refresh_request_headers(), ) - response_json = None - json_exception = None - try: - response_json = response.json() - except Exception as e: - # if the json could not be parsed, save the exception to raise it later - json_exception = e - - try: - if response_json: - # extract the access token and add to secrets to avoid logging the raw value - access_key = self._extract_access_token(response_json) - if access_key: - add_to_secrets(access_key) - except ResponseKeysMaxRecurtionReached as e: - # could not find the access token in the response, so do nothing - pass - - # log the response even if the request failed for troubleshooting purposes + if not response.ok: + # log the response even if the request failed for troubleshooting purposes + self._log_response(response) + response.raise_for_status() + + response_json = response.json() + + # extract the access token and add to secrets to avoid logging the raw value + access_key = self._extract_access_token(response_json) + if access_key: + add_to_secrets(access_key) + self._log_response(response) - response.raise_for_status() - - if json_exception: - raise json_exception return response_json except requests.exceptions.RequestException as e: From b24db1b537b3f73f593b64dc2cf147618ce77847 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 3 Jul 2025 13:47:00 -0700 Subject: [PATCH 7/8] handle max recursion error --- .../http/requests_native_auth/abstract_oauth.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py index 710fdf82d..f0a9cc3e1 100644 --- a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +++ b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py @@ -225,10 +225,14 @@ def _make_handled_request(self) -> Any: response_json = response.json() - # extract the access token and add to secrets to avoid logging the raw value - access_key = self._extract_access_token(response_json) - if access_key: - add_to_secrets(access_key) + try: + # extract the access token and add to secrets to avoid logging the raw value + access_key = self._extract_access_token(response_json) + if access_key: + add_to_secrets(access_key) + except ResponseKeysMaxRecurtionReached as e: + # could not find the access token in the response, so do nothing + pass self._log_response(response) From 503dac613beb26c6627cc0a24a48c0f0509732dc Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 3 Jul 2025 15:12:11 -0700 Subject: [PATCH 8/8] ruff format --- .../sources/streams/http/requests_native_auth/abstract_oauth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py index f0a9cc3e1..e7a4477c2 100644 --- a/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +++ b/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py @@ -233,7 +233,7 @@ def _make_handled_request(self) -> Any: except ResponseKeysMaxRecurtionReached as e: # could not find the access token in the response, so do nothing pass - + self._log_response(response) return response_json