Initial commit of template

LICENSE

@@ -1,7 +1,7 @@
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -618,44 +618,3 @@ copy of the Program in return for a fee.
 
                      END OF TERMS AND CONDITIONS
 
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-state the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published
-    by the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-Also add information on how to contact you by electronic and paper mail.
-
-  If your software can interact with users remotely through a computer
-network, you should also make sure that it provides a way for users to
-get its source.  For example, if your program is a web application, its
-interface could display a "Source" link that leads users to an archive
-of the code.  There are many ways you could offer source, and different
-solutions will be better for different programs; see section 13 for the
-specific requirements.
-
-  You should also get your employer (if you work as a programmer) or school,
-if any, to sign a "copyright disclaimer" for the program, if necessary.
-For more information on this, and how to apply and follow the GNU AGPL, see
-<https://www.gnu.org/licenses/>.

MANIFEST.in

@@ -0,0 +1,2 @@
+include LICENSE
+include tap_pardot/schemas/*.json

sample_config.json

@@ -0,0 +1,5 @@
+{
+  "username": "my_username",
+  "password": "my_password",
+  "start_date": "2017-01-01T00:00:00Z"
+}

setup.cfg

@@ -0,0 +1,2 @@
+[metadata]
+description-file = README.md

setup.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+from setuptools import setup
+
+setup(
+    name="tap-pardot",
+    version="0.1.0",
+    description="Singer.io tap for extracting data",
+    author="Stitch",
+    url="http://singer.io",
+    classifiers=["Programming Language :: Python :: 3 :: Only"],
+    py_modules=["tap_pardot"],
+    install_requires=[
+        "singer-python>=5.0.12",
+        "requests",
+    ],
+    entry_points="""
+    [console_scripts]
+    tap-pardot=tap_pardot:main
+    """,
+    packages=["tap_pardot"],
+    package_data = {
+        "schemas": ["tap_pardot/schemas/*.json"]
+    },
+    include_package_data=True,
+)

tap_pardot/__init__.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+import os
+import json
+import singer
+from singer import utils, metadata
+
+REQUIRED_CONFIG_KEYS = ["start_date", "username", "password"]
+LOGGER = singer.get_logger()
+
+def get_abs_path(path):
+    return os.path.join(os.path.dirname(os.path.realpath(__file__)), path)
+
+# Load schemas from schemas folder
+def load_schemas():
+    schemas = {}
+
+    for filename in os.listdir(get_abs_path('schemas')):
+        path = get_abs_path('schemas') + '/' + filename
+        file_raw = filename.replace('.json', '')
+        with open(path) as file:
+            schemas[file_raw] = json.load(file)
+
+    return schemas
+
+def discover():
+    raw_schemas = load_schemas()
+    streams = []
+
+    for schema_name, schema in raw_schemas.items():
+
+        # TODO: populate any metadata and stream's key properties here..
+        stream_metadata = []
+        stream_key_properties = []
+
+        # create and add catalog entry
+        catalog_entry = {
+            'stream': schema_name,
+            'tap_stream_id': schema_name,
+            'schema': schema,
+            'metadata' : [],
+            'key_properties': []
+        }
+        streams.append(catalog_entry)
+
+    return {'streams': streams}
+
+def get_selected_streams(catalog):
+    '''
+    Gets selected streams.  Checks schema's 'selected' first (legacy)
+    and then checks metadata (current), looking for an empty breadcrumb
+    and mdata with a 'selected' entry
+    '''
+    selected_streams = []
+    for stream in catalog.streams:
+        stream_metadata = metadata.to_map(stream.metadata)
+        # stream metadata will have an empty breadcrumb
+        if metadata.get(stream_metadata, (), "selected"):
+            selected_streams.append(stream.tap_stream_id)
+
+    return selected_streams
+
+def sync(config, state, catalog):
+
+    selected_stream_ids = get_selected_streams(catalog)
+
+    # Loop over streams in catalog
+    for stream in catalog.streams:
+        stream_id = stream.tap_stream_id
+        stream_schema = stream.schema
+        if stream_id in selected_stream_ids:
+            # TODO: sync code for stream goes here...
+            LOGGER.info('Syncing stream:' + stream_id)
+    return
+
+@utils.handle_top_exception(LOGGER)
+def main():
+
+    # Parse command line arguments
+    args = utils.parse_args(REQUIRED_CONFIG_KEYS)
+
+    # If discover flag was passed, run discovery mode and dump output to stdout
+    if args.discover:
+        catalog = discover()
+        print(json.dumps(catalog, indent=2))
+    # Otherwise run in sync mode
+    else:
+        if args.catalog:
+            catalog = args.catalog
+        else:
+            catalog =  discover()
+
+        sync(args.config, args.state, catalog)
+
+if __name__ == "__main__":
+    main()

tap_pardot/schemas/sample_stream.json

@@ -0,0 +1,19 @@
+{
+    "type": ["null", "object"],
+    "additionalProperties": false,
+    "properties": {
+	"string_field": {
+	    "type": ["null", "string"]
+	},
+      	"datetime_field": {
+            "type": ["null", "string"],
+            "format": "date-time"
+	},
+  	"integer_field": {
+	    "type": ["null", "integer"]
+        },
+      	"double_field": {
+	    "type": ["null", "number"]
+        }
+    }
+}