From fd3ede55d8d6997c32c6423873b4b752522b0952 Mon Sep 17 00:00:00 2001 From: jack-cloud-platform <38191888+jack-cloud-platform@users.noreply.github.com> Date: Wed, 27 Oct 2021 12:09:00 +0100 Subject: [PATCH] Added optional use of new v1 StatusCake API (#10) * Added optional use of v1 StatusCake API * Resolved linting errors --- README.md | 32 ++++++-- chart/status-cake-exporter/Tiltfile | 2 +- .../templates/deployment.yaml | 8 ++ chart/status-cake-exporter/values.yaml | 4 + exporter/app.py | 6 +- exporter/collectors/test_collector.py | 82 ++++++++++--------- exporter/status_cake_client/base.py | 22 +++-- exporter/status_cake_client/maintenance.py | 20 +++-- exporter/status_cake_client/tests.py | 42 +++++----- exporter/utilities/arguments.py | 20 ++++- requirements.txt | 3 +- 11 files changed, 155 insertions(+), 86 deletions(-) diff --git a/README.md b/README.md index 650076d..d4c6a1f 100644 --- a/README.md +++ b/README.md @@ -22,13 +22,15 @@ http://status-cake-exporter.default.svc:8000 ## Usage -| Setting | Required | Default | -|----------|----------|---------| -| USERNAME | Yes | Null | -| API_KEY | Yes | Null | -| TAGS | No | Null | -| LOG_LEVEL| No | info | -| PORT | No | 8000 | +| Setting | Required | Default | +|--------------------------------------|----------|---------| +| USE_V1_UPTIME_ENDPOINTS | No | False | +| USE_V1_MAINTENANCE_WINDOWS_ENDPOINTS | No | False | +| USERNAME | Yes | Null | +| API_KEY | Yes | Null | +| TAGS | No | Null | +| LOG_LEVEL | No | info | +| PORT | No | 8000 | ### Docker @@ -57,6 +59,8 @@ override environment variables which override defaults. optional arguments: -h, --help show this help message and exit + --use_v1_uptime_endpoints true Boolean for using v1 uptime endpoints [env var: USE_V1_UPTIME_ENDPOINTS] + --use_v1_maintenance_windows_endpoints true Boolean for using v1 maintenance windows endpoints [env var: USE_V1_MAINTENANCE_WINDOWS_ENDPOINTS] --username USERNAME Username for the account [env var: USERNAME] --api-key API_KEY API key for the account [env var: API_KEY] --tests.tags TAGS A comma separated list of tags used to filter tests returned from the api [env var: TAGS] @@ -64,6 +68,18 @@ optional arguments: --port The TCP port to start the web server on [env var: PORT] ``` +## V1 API +StatusCake have a new v1 API with documentation available at https://www.statuscake.com/api/v1/, deprecating the legacy API https://www.statuscake.com/api/. + +The new `Get all uptime tests` endpoint https://www.statuscake.com/api/v1/#operation/list-uptime-tests provides paged responses to get all tests, overcoming the limit of only 100 tests in the response from the legacy API https://www.statuscake.com/api/Tests/Get%20All%20Tests.md + +Environment variables `USE_V1_UPTIME_ENDPOINTS` and `USE_V1_MAINTENANCE_WINDOWS_ENDPOINTS` are used to enable use of the v1 API. + +### Maintenance Windows endpoints +Endpoints of the new v1 API are available to be used by all accounts with the exception of the maintenance windows endpoints, from https://www.statuscake.com/api/v1/#tag/uptime: +>NOTE: the API endpoints concerned with maintenance windows will only work with accounts registed to use the newer version of maintenance windows. This version of maintenance windows is incompatible with the original version and all existing windows will require migrating to be further used. Presently a tool to automate the migration of maintenance windows is under development. +Similarly, if an account is registered to use the newer version of maintenance windows, the legacy API's maintenance windows endpoints cannot be used. + ## Metrics | Name| Type | Description | @@ -94,7 +110,7 @@ Data collected by Prometheus can be easily surfaced in Grafana. Using the [Statusmap panel](https://grafana.com/grafana/plugins/flant-statusmap-panel) by [flant](https://github.com/flant/grafana-statusmap) you can create a basic status visualization based on uptime percentage: -![grafan](examples/grafana.png) +![grafana](examples/grafana.png) ### PromQL diff --git a/chart/status-cake-exporter/Tiltfile b/chart/status-cake-exporter/Tiltfile index 48d4294..a9ebbe3 100644 --- a/chart/status-cake-exporter/Tiltfile +++ b/chart/status-cake-exporter/Tiltfile @@ -1,7 +1,7 @@ docker_build('status-cake-exporter:dev', '../../') # If not using a standard local dev name, specify your k8s context here #allow_k8s_contexts('microk8s') -k8s_yaml(helm('.', values='values.yaml', set=['statuscake.logLevel=debug', 'image.repository=status-cake-exporter', 'image.tag=dev', 'statuscake.username=', 'statuscake.apiKey=', 'statuscake.tags=firstTag,secondTag'])) +k8s_yaml(helm('.', values='values.yaml', set=['statuscake.logLevel=debug', 'image.repository=status-cake-exporter', 'image.tag=dev', 'statuscake.useV1UptimeEndpoints=', 'statuscake.useV1MaintenanceWindowsEndpoints=', 'statuscake.username=', 'statuscake.apiKey=', 'statuscake.tags=firstTag,secondTag'])) watch_file('.') watch_file('../../Dockerfile') watch_file('../../exporter') diff --git a/chart/status-cake-exporter/templates/deployment.yaml b/chart/status-cake-exporter/templates/deployment.yaml index 7065086..9814671 100644 --- a/chart/status-cake-exporter/templates/deployment.yaml +++ b/chart/status-cake-exporter/templates/deployment.yaml @@ -24,6 +24,14 @@ spec: ports: - containerPort: {{ .Values.service.port }} env: +{{- if .Values.statuscake.useV1UptimeEndpoints }} + - name: USE_V1_UPTIME_ENDPOINTS + value: {{ .Values.statuscake.useV1UptimeEndpoints }} +{{- end }} +{{- if .Values.statuscake.useV1MaintenanceWindowsEndpoints }} + - name: USE_V1_MAINTENANCE_WINDOWS_ENDPOINTS + value: {{ .Values.statuscake.useV1MaintenanceWindowsEndpoints }} +{{- end }} - name: USERNAME valueFrom: secretKeyRef: diff --git a/chart/status-cake-exporter/values.yaml b/chart/status-cake-exporter/values.yaml index 2e4600b..af18053 100644 --- a/chart/status-cake-exporter/values.yaml +++ b/chart/status-cake-exporter/values.yaml @@ -5,6 +5,10 @@ image: pullSecrets: [] statuscake: + # optional: a boolean format string for using the uptime endpoints of the v1 API + # useV1UptimeEndpoints: + # optional: a boolean format string for using the maintenance windows endpoints of the v1 API + # useV1MaintenanceWindowsEndpoints: # REQUIRED: username to use when connecting to statuscake username: "" # REQUIRED: apikey to use when connecting to statuscake diff --git a/exporter/app.py b/exporter/app.py index f3e500a..63b0e82 100644 --- a/exporter/app.py +++ b/exporter/app.py @@ -23,7 +23,11 @@ logger.info("Registering collectors.") REGISTRY.register(test_collector.TestCollector( - args.username, args.api_key, args.tags)) + args.use_v1_uptime_endpoints, + args.use_v1_maintenance_windows_endpoints, + args.username, + args.api_key, + args.tags)) while True: time.sleep(1) diff --git a/exporter/collectors/test_collector.py b/exporter/collectors/test_collector.py index ec29e29..873c92d 100644 --- a/exporter/collectors/test_collector.py +++ b/exporter/collectors/test_collector.py @@ -3,57 +3,56 @@ import sys import logging from prometheus_client.core import GaugeMetricFamily +from setuptools import distutils from status_cake_client import tests as t from status_cake_client import maintenance as m logger = logging.getLogger("test_collector") -def parse_test_response(r, m): +def parse_test_response(use_v1_uptime_endpoints, tests, m_test_id_flat_list): t = [] try: - tests = r.json() + if use_v1_uptime_endpoints: + for i in tests: + t.append( + { + "test_id": str(i['id']), + "test_type": i['test_type'], + "test_name": i['name'], + "test_url": i['website_url'], + "test_status_int": str(1 if (i["status"] == "up") else 0), + "test_uptime_percent": str(i['uptime']), + "maintenance_status_int": str(1 if (str(i["id"])) in m_test_id_flat_list else 0) + } + ) + else: + for i in tests: + t.append( + { + "test_id": str(i['TestID']), + "test_type": i['TestType'], + "test_name": i['WebsiteName'], + "test_url": i['WebsiteURL'], + "test_status_int": str(1 if (i["Status"] == "Up") else 0), + "test_uptime_percent": str(i['Uptime']), + "maintenance_status_int": str(1 if (str(i["TestID"])) in m_test_id_flat_list else 0) + } + ) + + return t + except Exception as e: logger.error(f"Could not parse test data, exception: {e}") - logger.error(f"Test data was:\n{r}") + logger.error(f"Test data was:\n{tests}") sys.exit(1) - for i in tests: - t.append( - { - "test_id": str(i['TestID']), - "test_type": i['TestType'], - "test_name": i['WebsiteName'], - "test_url": i['WebsiteURL'], - "test_status_int": str(1 if (i["Status"] == "Up") else 0), - "test_uptime_percent": str(i['Uptime']), - "maintenance_status_int": str(1 if (str(i["TestID"])) in m else 0) - } - ) - - return t - - -def parse_test_details_response(r): - t = [] - for i in r: - t.append( - { - "test_id": str(i['TestID']), - "test_status_string": i['Status'], - "test_status_int": str(1 if (i["Status"] == "Up") else 0), - "test_uptime_percent": str(i['Uptime']), - "test_last_tested": i['LastTested'], - "test_processing": i['Processing'], - "test_down_times": str(i['DownTimes']) - } - ) - - return t class TestCollector(object): - def __init__(self, username, api_key, tags): + def __init__(self, use_v1_uptime_endpoints, use_v1_maintenance_windows_endpoints, username, api_key, tags): + self.use_v1_uptime_endpoints = bool(distutils.util.strtobool(use_v1_uptime_endpoints)) + self.use_v1_maintenance_windows_endpoints = bool(distutils.util.strtobool(use_v1_maintenance_windows_endpoints)) self.username = username self.api_key = api_key self.tags = tags @@ -64,7 +63,7 @@ def collect(self): try: - maintenance = m.get_maintenance(self.api_key, self.username) + maintenance = m.get_maintenance(self.use_v1_maintenance_windows_endpoints, self.api_key, self.username) try: maintenance_data = maintenance.json()['data'] except Exception as e: @@ -74,14 +73,17 @@ def collect(self): logger.debug(f"Maintenance response:\n{maintenance_data}") # Grab the test_ids from the response - m_test_id_list = [i['all_tests'] for i in maintenance_data] + if self.use_v1_maintenance_windows_endpoints: + m_test_id_list = [i['tests'] for i in maintenance_data] + else: + m_test_id_list = [i['all_tests'] for i in maintenance_data] # Flatten the test_ids into a list m_test_id_flat_list = [item for sublist in m_test_id_list for item in sublist] logger.info(f"Found {len(m_test_id_flat_list)} tests that are in maintenance.") - tests = t.get_tests(self.api_key, self.username, self.tags) - parsed_tests = parse_test_response(tests, m_test_id_flat_list) + tests = t.get_tests(self.use_v1_uptime_endpoints, self.api_key, self.username, self.tags) + parsed_tests = parse_test_response(self.use_v1_uptime_endpoints, tests, m_test_id_flat_list) logger.info(f"Publishing {len(parsed_tests)} tests.") # status_cake_test_info - gauge diff --git a/exporter/status_cake_client/base.py b/exporter/status_cake_client/base.py index 958cfd8..be28b14 100644 --- a/exporter/status_cake_client/base.py +++ b/exporter/status_cake_client/base.py @@ -4,20 +4,28 @@ import requests STATUS_CAKE_BASE_URL = "https://app.statuscake.com/API/" +V1_STATUS_CAKE_BASE_URL = "https://api.statuscake.com/v1/" logger = logging.getLogger(__name__) -def get(apikey, username, endpoint, params={}): +def get(use_v1_api, apikey, username, endpoint, params={}): - request_url = f"{STATUS_CAKE_BASE_URL}{endpoint}" + if use_v1_api: + headers = { + "Authorization": "Bearer %s" % apikey + } + BASE_URL = V1_STATUS_CAKE_BASE_URL + else: + headers = { + "API": apikey, + "Username": username + } + BASE_URL = STATUS_CAKE_BASE_URL - logger.debug(f"Starting request: {request_url} {endpoint} {params}") + request_url = f"{BASE_URL}{endpoint}" - headers = { - "API": apikey, - "Username": username - } + logger.debug(f"Starting request: {request_url} {endpoint} {params}") response = requests.get(url=request_url, params=params, headers=headers) response.raise_for_status() diff --git a/exporter/status_cake_client/maintenance.py b/exporter/status_cake_client/maintenance.py index 3e2e64e..b406e7b 100644 --- a/exporter/status_cake_client/maintenance.py +++ b/exporter/status_cake_client/maintenance.py @@ -9,16 +9,22 @@ logger = logging.getLogger(__name__) -def get_maintenance(apikey, username, state="ACT"): - endpoint = "Maintenance" - params = { - "state": state - } +def get_maintenance(use_v1_maintenance_windows_endpoints, apikey, username): + if use_v1_maintenance_windows_endpoints: + endpoint = "maintenance-windows" + params = { + "state": "active" + } + else: + endpoint = "Maintenance" + params = { + "state": "ACT" + } try: - response = get(apikey, username, endpoint, params) + response = get(use_v1_maintenance_windows_endpoints, apikey, username, endpoint, params) except requests.exceptions.HTTPError as e: - if e.response.status_code == 404: + if not(use_v1_maintenance_windows_endpoints) and e.response.status_code == 404: logger.info("Currently no active maintenance.") response = e.response else: diff --git a/exporter/status_cake_client/tests.py b/exporter/status_cake_client/tests.py index e8c3582..84c4422 100644 --- a/exporter/status_cake_client/tests.py +++ b/exporter/status_cake_client/tests.py @@ -6,24 +6,28 @@ logger = logging.getLogger(__name__) -def get_tests(apikey, username, tags=""): - endpoint = "Tests" - params = { - "tags": tags - } - response = get(apikey, username, endpoint, params) +def get_tests(use_v1_uptime_endpoints, apikey, username, tags=""): + if use_v1_uptime_endpoints: + page = 1 + endpoint = "uptime" + params = { + "tags": tags, + "page": page + } + response = get(use_v1_uptime_endpoints, apikey, username, endpoint, params) + tests = response.json()['data'] + while (page < (response.json()['metadata']['page_count'])): + page += 1 + params["page"] = page + response = get(use_v1_uptime_endpoints, apikey, username, endpoint, params) + tests += response.json()['data'] + else: + endpoint = "Tests" + params = { + "tags": tags + } + response = get(use_v1_uptime_endpoints, apikey, username, endpoint, params) + tests = response.json()['data'] logger.debug(f"Request response:\n{response.content}") - return response - - -def get_test_details(apikey, username, test_id): - endpoint = "Tests/Details/" - params = { - "TestID": test_id - } - - response = get(apikey, username, endpoint, params) - logger.debug(f"Request response:\n{response.content}") - - return response + return tests diff --git a/exporter/utilities/arguments.py b/exporter/utilities/arguments.py index 1e0033c..51ed0d7 100644 --- a/exporter/utilities/arguments.py +++ b/exporter/utilities/arguments.py @@ -7,6 +7,24 @@ def get_args(): parser = configargparse.ArgParser() + parser.add("--use_v1_uptime_endpoints", + dest="use_v1_uptime_endpoints", + env_var="USE_V1_UPTIME_ENDPOINTS", + default="false", + type=str.lower, + choices={'false', 'f', '0', 'off', 'no', 'n', 'off', + 'true', 't', '1', 'on', 'yes', 'y', 't', 'true', 'on'}, + help='Boolean format string for using the uptime endpoints of the v1 API') + + parser.add("--use_v1_maintenance_windows_endpoints", + dest="use_v1_maintenance_windows_endpoints", + env_var="USE_V1_MAINTENANCE_WINDOWS_ENDPOINTS", + default="false", + type=str.lower, + choices={'false', 'f', '0', 'off', 'no', 'n', 'off', + 'true', 't', '1', 'on', 'yes', 'y', 't', 'true', 'on'}, + help='Boolean format string for using the maintenance windows endpoints of the v1 API') + parser.add("--username", dest="username", env_var="USERNAME", @@ -44,7 +62,7 @@ def get_args(): sys.exit(1) if args.api_key is None: - print("Required argument --username is missing") + print("Required argument --api_key is missing") print(parser.print_help()) sys.exit(1) diff --git a/requirements.txt b/requirements.txt index da4f4e8..d0e61b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ prometheus-client==0.7.1 requests==2.22.0 ConfigArgParse==0.14.0 - - +setuptools==58.3.0 \ No newline at end of file