TDL-16139 Support of service account authentication

.circleci/config.yml

@@ -15,12 +15,12 @@ jobs:
             pip install -U 'pip<19.2' 'setuptools<51.0.0'
             pip install .[dev]
             pip install pytest-cov
-      # TODO: Fails pylint a lot, skipping for now (https://stitchdata.atlassian.net/browse/SRCE-4606)
-      #- run:
-      #    name: 'pylint tap'
-      #    command: |
-      #      source /usr/local/share/virtualenvs/tap-mixpanel/bin/activate
-      #      pylint tap_mixpanel -d 'broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-class-docstring,missing-function-docstring,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,wrong-spelling-in-comment,wrong-spelling-in-docstring,too-many-public-methods'
+      - run:
+         name: 'pylint tap'
+         command: |
+           source /usr/local/share/virtualenvs/tap-mixpanel/bin/activate
+           pip install pylint
+           pylint tap_mixpanel -d 'broad-except,chained-comparison,empty-docstring,fixme,invalid-name,line-too-long,missing-module-docstring,no-else-raise,no-else-return,too-few-public-methods,too-many-arguments,too-many-branches,too-many-lines,too-many-locals,ungrouped-imports,too-many-public-methods,protected-access,too-many-statements,not-an-iterable,too-many-instance-attributes'
       - run:
           name: 'JSON Validator'
           command: |
@@ -30,6 +30,7 @@ jobs:
           name: 'Unit Tests'
           command: |
             source /usr/local/share/virtualenvs/tap-mixpanel/bin/activate
+            pip install coverage parameterized
             python -m pytest --junitxml=junit/test-result.xml --cov=tap_mixpanel --cov-report=html tests/unittests/
       - store_test_results:
           path: test_output/report.xml

README.md

@@ -133,6 +133,9 @@ More details may be found in the [Mixpanel API Authentication](https://developer
    - `start_date` - the default value to use if no bookmark exists for an endpoint (rfc3339 date string)
    - `user_agent` (string, optional): Process and email for API logging purposes. Example: `tap-mixpanel <api_user_email@your_company.com>`
    - `api_secret` (string, `ABCdef123`): an API secret for each project in Mixpanel. This can be found in the Mixpanel Console, upper-right Settings (gear icon), Organization Settings > Projects and in the Access Keys section. For this tap, only the api_secret is needed (the api_key is legacy and the token is used only for uploading data). Each Mixpanel project has a different api_secret; therefore each Singer tap pipeline instance is for a single project.
+   - `service_account_username` (string, `username12`): Username of the service account.
+   - `service_account_secret` (string, `ABCdef123`): Secret of the service account.
+   - `project_id` (string, `10451202`): Id of the project which is connected to the provided service account.
    - `date_window_size` (integer, `30`): Number of days for date window looping through transactional endpoints with from_date and to_date. Default date_window_size is 30 days. Clients with large volumes of events may want to decrease this to 14, 7, or even down to 1-2 days.
    - `attribution_window` (integer, `5`): Latency minimum number of days to look-back to account for delays in attributing accurate results. [Default attribution window is 5 days](https://help.mixpanel.com/hc/en-us/articles/115004616486-Tracking-If-Users-Are-Offline).
    - `project_timezone` (string like `US/Pacific`): Time zone in which integer date times are stored. The project timezone may be found in the project settings in the Mixpanel console. [More info about timezones](https://help.mixpanel.com/hc/en-us/articles/115004547203-Manage-Timezones-for-Projects-in-Mixpanel). 

tap_mixpanel/__init__.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
 
-import sys
 import json
-import argparse
-from datetime import datetime, timedelta, date
+import sys
+from datetime import timedelta
+
 import singer
-from singer import metadata, utils
-from singer.utils import strptime_to_utc, strftime
+from singer import utils
+from singer.utils import strftime, strptime_to_utc
+
 from tap_mixpanel.client import MixpanelClient
 from tap_mixpanel.discover import discover
 from tap_mixpanel.sync import sync
@@ -15,73 +16,89 @@
 
 REQUEST_TIMEOUT = 300
 REQUIRED_CONFIG_KEYS = [
-    'project_timezone',
-    'api_secret',
-    'attribution_window',
-    'start_date',
-    'user_agent'
+    "project_timezone",
+    "attribution_window",
+    "start_date",
+    "user_agent",
 ]
 
 
 def do_discover(client, properties_flag):
-    LOGGER.info('Starting discover')
+    """Call the discovery function.
+
+    Args:
+        client (MixpanelClient): Client object to make http calls.
+        properties_flag (str): Setting this argument to `true` ensures that new properties on
+                               events and engage records are captured.
+    """
+    LOGGER.info("Starting discover")
     catalog = discover(client, properties_flag)
     json.dump(catalog.to_dict(), sys.stdout, indent=2)
-    LOGGER.info('Finished discover')
+    LOGGER.info("Finished discover")
 
 
 @singer.utils.handle_top_exception(LOGGER)
 def main():
+    """
+    Run discover mode or sync mode.
+    """
     parsed_args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)
 
-    start_date = parsed_args.config['start_date']
+    start_date = parsed_args.config["start_date"]
     # Set request timeout to config param `request_timeout` value.
     # If value is 0, "0", "" or not passed then it sets default to 300 seconds.
-    config_request_timeout = parsed_args.config.get('request_timeout')
+    config_request_timeout = parsed_args.config.get("request_timeout")
     if config_request_timeout and float(config_request_timeout):
         request_timeout = float(config_request_timeout)
     else:
         request_timeout = REQUEST_TIMEOUT
 
     start_dttm = strptime_to_utc(start_date)
     now_dttm = utils.now()
-    if parsed_args.config.get('end_date'):
-        now_dttm = strptime_to_utc(parsed_args.config.get('end_date'))
+    if parsed_args.config.get("end_date"):
+        now_dttm = strptime_to_utc(parsed_args.config.get("end_date"))
     delta_days = (now_dttm - start_dttm).days
     if delta_days >= 365:
         delta_days = 365
         start_date = strftime(now_dttm - timedelta(days=delta_days))
-        LOGGER.warning("start_date greater than 1 year maxiumum for API.")
+        LOGGER.warning("start_date greater than 1 year maximum for API.")
         LOGGER.warning("Setting start_date to 1 year ago, %s", start_date)
 
-    #Check support for EU endpoints
-    if str(parsed_args.config.get('eu_residency')).lower() == "true":
+    # Check support for EU endpoints
+    if str(parsed_args.config.get("eu_residency")).lower() == "true":
         api_domain = "eu.mixpanel.com"
     else:
         api_domain = "mixpanel.com"
 
-    with MixpanelClient(parsed_args.config['api_secret'],
-                        api_domain,
-                        request_timeout,
-                        parsed_args.config['user_agent']) as client:
+    with MixpanelClient(
+        parsed_args.config.get("api_secret"),
+        parsed_args.config.get("service_account_username"),
+        parsed_args.config.get("service_account_secret"),
+        parsed_args.config.get("project_id"),
+        api_domain,
+        request_timeout,
+        parsed_args.config["user_agent"],
+    ) as client:
 
         state = {}
         if parsed_args.state:
             state = parsed_args.state
 
         config = parsed_args.config
-        properties_flag = config.get('select_properties_by_default')
+        properties_flag = config.get("select_properties_by_default")
 
         if parsed_args.discover:
             client.__api_domain = api_domain
             do_discover(client, properties_flag)
         elif parsed_args.catalog:
-            sync(client=client,
-                 config=config,
-                 catalog=parsed_args.catalog,
-                 state=state,
-                 start_date=start_date)
+            sync(
+                client=client,
+                config=config,
+                catalog=parsed_args.catalog,
+                state=state,
+                start_date=start_date,
+            )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()