updated prospects stream sync function (#42)

* updated prospects stream sync function
* updated cci config

.circleci/config.yml

@@ -0,0 +1,51 @@
+version: 2.1
+jobs:
+  build:
+    docker:
+      - image: 218546966473.dkr.ecr.us-east-1.amazonaws.com/circle-ci:stitch-tap-tester
+    steps:
+      - checkout
+      - run:
+          name: 'Setup virtual env'
+          command: |
+            python3 -mvenv /usr/local/share/virtualenvs/tap-pardot
+            source /usr/local/share/virtualenvs/tap-pardot/bin/activate
+            pip install -U 'pip==22.2.2' 'setuptools==65.3.0'
+            pip install -U pylint
+            pip install .[dev]
+      - run:
+          name: 'JSON Validator'
+          command: |
+            source /usr/local/share/virtualenvs/tap-tester/bin/activate
+            stitch-validate-json tap_pardot/schemas/*.json
+      - run:
+          name: 'pylint'
+          command: |
+            source /usr/local/share/virtualenvs/tap-pardot/bin/activate
+            pylint tap_pardot -d C,R,W
+      - run:
+          name: 'Integration Tests'
+          command: |
+            aws s3 cp s3://com-stitchdata-dev-deployment-assets/environments/tap-tester/tap_tester_sandbox dev_env.sh
+            source dev_env.sh
+            source /usr/local/share/virtualenvs/tap-tester/bin/activate
+            run-test --tap=tap-pardot tests
+
+
+workflows:
+  version: 2
+  commit:
+    jobs:
+      - build:
+          context: circleci-user
+  build_daily:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - build:
+          context: circleci-user

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 1.4.5
+  * Prospects Stream Fix Pagination [#42](https://github.com/singer-io/tap-pardot/pull/42)
+
 ## 1.4.4
   * Dependabot update [#41](https://github.com/singer-io/tap-pardot/pull/41)
 

setup.py

@@ -3,7 +3,7 @@
 
 setup(
     name="tap-pardot",
-    version="1.4.4",
+    version="1.4.5",
     description="Singer.io tap for extracting data",
     author="Stitch",
     url="http://singer.io",

tap_pardot/streams.py

@@ -326,7 +326,7 @@ def get_parent_ids(self, parent):
 
     def sync(self):
         self.pre_sync()
-
+        # pylint: disable=E1102
         parent = self.parent_class(
             self.client, self.config, self.parent_bookmark, emit=False
         )
@@ -376,6 +376,24 @@ class Prospects(UpdatedAtReplicationStream):
 
     is_dynamic = False
 
+    def get_records(self):
+        offset = 0
+        params = self.get_params()
+        while True:
+            params["offset"] = offset
+            data = self.client.get(self.endpoint, **params)
+            records = data["result"]
+            if not records:
+                break
+            offset += 200
+            yield from records[self.data_key]
+
+    def sync(self):
+        self.pre_sync()
+        for rec in self.sync_page():
+            yield rec
+        self.post_sync()
+
 
 class Opportunities(NoUpdatedAtSortingStream):
     stream_name = "opportunities"