[TDL-25850] Add backoff for client error 406 (#181)

* Bump version 2.1.1 * * Add backoff for client error 406 * * Fix integration tests --------- Co-authored-by: RushiT0122 <[email protected]> Co-authored-by: Rushikesh Todkar <[email protected]>
singer-io · Feb 3, 2025 · dfd0c69 · dfd0c69
1 parent 48754f7
commit dfd0c69
Show file tree

Hide file tree

Showing 8 changed files with 49 additions and 16 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,8 +1,11 @@
 # Changelog
 
-## 2.0.2
+## 2.1.1
+  - Add backoff for 406 Client Error [#184](https://github.com/singer-io/tap-salesforce/pulls)
+
+## 2.1.0
   - Upgrade salesforce API version `52.0` to `61.0`, details [#184](https://github.com/singer-io/tap-salesforce/pulls)
-  
+
 ## 2.0.1
   * Fix `_can_pk_chunk_job` condition, details: [#176](https://github.com/singer-io/tap-salesforce/pull/176)
 

diff --git a/setup.py b/setup.py
@@ -3,14 +3,14 @@
 from setuptools import setup
 
 setup(name='tap-salesforce',
-      version='2.1.0',
+      version='2.1.1',
       description='Singer.io tap for extracting data from the Salesforce API',
       author='Stitch',
       url='https://singer.io',
       classifiers=['Programming Language :: Python :: 3 :: Only'],
       py_modules=['tap_salesforce'],
       install_requires=[
-          'requests==2.31.0',
+          'requests==2.32.3',
           'singer-python==5.13.0',
           'xmltodict==0.11.0'
       ],

diff --git a/tap_salesforce/salesforce/__init__.py b/tap_salesforce/salesforce/__init__.py
@@ -13,7 +13,8 @@
 from tap_salesforce.salesforce.rest import Rest, API_VERSION
 from tap_salesforce.salesforce.exceptions import (
     TapSalesforceException,
-    TapSalesforceQuotaExceededException)
+    TapSalesforceQuotaExceededException,
+    Client406Error)
 
 LOGGER = singer.get_logger()
 
@@ -285,8 +286,8 @@ def check_rest_quota_usage(self, headers):
 
     # pylint: disable=too-many-arguments,too-many-positional-arguments
     @backoff.on_exception(backoff.expo,
-                          (requests.exceptions.ConnectionError, requests.exceptions.Timeout),
-                          max_tries=10,
+                          (requests.exceptions.ConnectionError, requests.exceptions.Timeout, Client406Error),
+                          max_tries=6,
                           factor=2,
                           on_backoff=log_backoff_attempt)
     def _make_request(self, http_method, url, headers=None, body=None, stream=False, params=None):
@@ -314,7 +315,8 @@ def _make_request(self, http_method, url, headers=None, body=None, stream=False,
             LOGGER.error('Took longer than %s seconds to hear from the server', request_timeout)
             raise timeout_err
 
-
+        if resp.status_code == 406:
+            raise Client406Error
 
         try:
             resp.raise_for_status()

diff --git a/tap_salesforce/salesforce/exceptions.py b/tap_salesforce/salesforce/exceptions.py
@@ -8,3 +8,6 @@ class TapSalesforceQuotaExceededException(TapSalesforceException):
 
 class TapSalesforceBulkAPIDisabledException(TapSalesforceException):
     pass
+
+class Client406Error(Exception):
+    pass
diff --git a/tests/base.py b/tests/base.py
@@ -551,9 +551,6 @@ def expected_metadata(self):
             'OrderShare': incremental_last_modified,
             'OrgDeleteRequest': default,  # new
             'OrgDeleteRequestShare': incremental_last_modified,  # new
-            'OrgMetric': default,  # new
-            'OrgMetricScanResult': default,  # new
-            'OrgMetricScanSummary': default,  # new
             'OrgWideEmailAddress': default,
             'Organization': default,
             'PackageLicense': default,  # new
@@ -1098,7 +1095,6 @@ def expected_metadata(self):
             'ExtlClntAppOauthSetAttr': default,
             'LocationShippingCarrierMethod': default,
             'DeliveryEstimationSetupHistory': incremental_created_date
-
         }
 
     def rest_only_streams(self):

diff --git a/tests/sfbase.py b/tests/sfbase.py
@@ -542,9 +542,6 @@ def expected_metadata():
             'OrderShare': incremental_last_modified,
             'OrgDeleteRequest': default,  # new
             'OrgDeleteRequestShare': incremental_last_modified,  # new
-            'OrgMetric': default,  # new
-            'OrgMetricScanResult': default,  # new
-            'OrgMetricScanSummary': default,  # new
             'OrgWideEmailAddress': default,
             'Organization': default,
             'PackageLicense': default,  # new

diff --git a/tests/test_salesforce_all_fields_non_custom_rest.py b/tests/test_salesforce_all_fields_non_custom_rest.py
@@ -31,9 +31,10 @@ def streams_to_selected_fields(self):
         return non_custom_fields
 
     def test_non_custom_fields(self):
+        excluded_fields = {'MlFeatureValueMetric'}
         for stream in self.streams_to_test():
             with self.subTest(stream=stream):
-                expected_non_custom_fields = self.selected_fields.get(stream,set())
+                expected_non_custom_fields = self.selected_fields.get(stream,set()) - excluded_fields
                 replicated_non_custom_fields = self.actual_fields.get(stream, set())
                 #Verify at least one non-custom field is replicated
                 self.assertGreater(len(replicated_non_custom_fields),0,

diff --git a/tests/unittests/test_multiline_critical_error_message.py b/tests/unittests/test_multiline_critical_error_message.py
@@ -27,3 +27,34 @@ def test_multiline_critical_error_message(self, mocked_main_impl, mocked_logger_
 
         # verify "LOGGER.critical" is called 5 times, as the error raised contains 5 lines
         self.assertEqual(mocked_logger_critical.call_count, 5)
+
+    @mock.patch("tap_salesforce.singer_utils.parse_args")
+    @mock.patch('tap_salesforce.salesforce.requests.Session.post')
+    def test_http_406_error_message(self, mocked_post, mocked_parse_args):
+
+        args = mock.MagicMock()
+        args.config = {
+            "refresh_token": "abc",
+            "client_id": "abc",
+            "client_secret": "abc",
+            "quota_percent_total": 10.1,
+            "quota_percent_per_run": 10.1,
+            "is_sandbox": True,
+            "start_date": "2020-02-04T07:46:29Z",
+            "api_type": "abc",
+            "lookback_window": "12"
+        }
+        mocked_parse_args.return_value = args
+
+        # Define the mock response
+        mock_response = mock.MagicMock()
+        mock_response.status_code = 406
+        mocked_post.return_value = mock_response
+
+
+        # verify "Exception" is raise on function call
+        with self.assertRaises(Exception):
+            main()
+
+        # verify "requests.Session.post" is called 6 times
+        self.assertEqual(mocked_post.call_count, 6)
-Original file line number
+Diff line change
@@ Expand Up @@
     class TapSalesforceBulkAPIDisabledException(TapSalesforceException):
         pass
+    class Client406Error(Exception):
+        pass