Initial commit

Author: kidk

.gitignore

@@ -0,0 +1,6 @@
+# CoScale CLI files
+api.conf
+coscale-cli-*
+
+# Vscode
+.vscode

stats.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+import json
+import requests
+from datetime import timedelta, datetime, tzinfo
+import time
+import calendar
+
+#
+# Configuration values
+#
+
+# Location of CoScale install
+ENV = "https://app.coscale.com/api/v1/app"
+
+# Name of the metric
+METRIC_NAME = "Docker container total cpu usage in cores"
+
+# Application data
+APP_ID = "000000-08e8-4f72-0000-220ff6600000"
+APP_TOKEN = "000000-ebd1-4880-0000-3f4c06500000"
+
+# Data to retrieve
+START_DATE = datetime(2018, 5, 25)
+END_DATE = datetime(2018, 5, 27)
+
+# Images to retrieve data for, these should match the ones in CoScale Docker group
+IMAGES = [
+    "docker.io/kidk/training-images-calc_words",
+    "docker.io/kidk/training-images-web",
+]
+
+#
+# DO NOT CHANGE ANYTHING BELOW THIS LINE
+#
+def dt2ts(dt):
+    """Converts a datetime object to UTC timestamp"""
+    return calendar.timegm(dt.utctimetuple())
+
+def debug(variable, fail=True):
+    print(json.dumps(variable, indent=2, sort_keys=True))
+    if fail:
+        exit(0)
+
+print("Retrieving data timerange from %s to %s" % (
+    START_DATE, END_DATE
+))
+START_TIME = dt2ts(START_DATE)
+END_TIME = dt2ts(END_DATE)
+
+class API:
+
+    token = None
+    app = None
+    env = None
+
+    def __init__(self, env, app, token):
+        r = requests.post(
+            "{env}/{app}/login/".format(env=env, app=app), data={"accessToken": token}
+        )
+        self.token = json.loads(r.text)["token"]
+        self.app = app
+        self.env = env
+
+    def do_post():
+        return None
+
+    def do_get():
+        return None
+
+    def get_metric_id(self, name):
+        r = requests.get(
+            "{env}/{app}/metrics/?selectByName={name}".format(
+                env=self.env, app=self.app, name=name
+            ),
+            headers={"HTTPAuthorization": self.token},
+            data={},
+        )
+        metrics = json.loads(r.text)
+
+        return metrics[0]["id"]
+
+    def get_servergroup_children(self, id, start, stop):
+        # Retrieve group children
+        r = requests.get(
+            "{env}/{app}/servergroups/{group_id}/servergroups/?start={start}&stop={stop}".format(
+                env=self.env, app=self.app, group_id=id, start=start, stop=stop
+            ),
+            headers={"HTTPAuthorization": self.token},
+        )
+
+        return json.loads(r.text)
+
+    def get_servergroup_servers(self, id, start, stop):
+        r = requests.get(
+            "{env}/{app}/servergroups/{group_id}/servers/?start={start}&stop={stop}".format(
+                env=self.env, app=self.app, group_id=id, start=start, stop=stop
+            ),
+            headers={"HTTPAuthorization": self.token},
+        )
+
+        return json.loads(r.text)
+
+    def get_servergroup(self, name):
+        # Retrieve group ID
+        r = requests.get(
+            "{env}/{app}/servergroups/?selectByName={name}".format(
+                env=self.env, app=self.app, name=name
+            ),
+            headers={"HTTPAuthorization": self.token},
+            data={},
+        )
+
+        return json.loads(r.text)[0]
+
+    def get_data(self, metric, servers, start, stop):
+        postdata = {
+            "start": range_start_time,
+            "stop": range_stop_time,
+            "ids": [
+                {
+                    "metricId": METRIC_ID,
+                    "dimensionsSpecs": [],
+                    "subjects": "%s" % servers,
+                    "viewtype": "AVG",
+                }
+            ],
+        }
+        r = requests.post(
+            "{env}/{app}/data/dimension/getCalculated/calculated/?function=summary&mode=average".format(
+                env=self.env, app=self.app
+            ),
+            headers={"HTTPAuthorization": self.token},
+            data={"data": json.dumps(postdata)},
+        )
+
+        return json.loads(r.text)
+
+    def get_all_servers_in_group_helper(self, group):
+        serverIds = group['serverIds']
+
+        for child_group in group['servergroups']:
+            serverIds = serverIds + self.get_all_servers_in_group_helper(child_group)
+
+        return serverIds
+
+    def get_all_servers_in_group(self, id, start, stop):
+        # Retrieve all containers
+        r = requests.get(
+            "{env}/{app}/servergroups/{group_id}/?expand=servergroups&expand=serverIds&start={start}&stop={stop}".format(
+                env=self.env, app=self.app, group_id=id, start=start, stop=stop
+            ),
+            headers={"HTTPAuthorization": self.token},
+        )
+        groups = json.loads(r.text)
+
+        # Get list of all container id's
+        serverIds = self.get_all_servers_in_group_helper(groups)
+
+        return list(set(serverIds))  # Remove duplicates
+
+
+# Init API and login
+API = API(ENV, SOURCE['id'], SOURCE['token'])
+
+# Retrieve the metric ID for metric name
+METRIC_ID = API.get_metric_id(METRIC_NAME)
+print("Retrieved metricId %s for '%s'" % (METRIC_ID, METRIC_NAME))
+
+# Retrieve Docker group children
+image_group = API.get_servergroup('Docker')
+images_group = API.get_servergroup_children(image_group['id'], start=START_TIME, stop=END_TIME)
+
+# Prepare list of all containers running for image
+containers = {}
+for group in images_group:
+    image_name = group["name"]
+    if image_name in IMAGES:
+        # Add element to containers list
+        if image_name not in containers:
+            containers[image_name] = {}
+
+        # Retrieve subgroups
+        subgroups = API.get_servergroup_children(group["id"], start=START_TIME, stop=END_TIME)
+
+        # Retrieve containers from subgroups
+        for subgroup in subgroups:
+            servers = API.get_servergroup_servers(subgroup['id'], START_TIME, END_TIME)
+
+            # Add servers to containers list
+            for server in servers:
+                server_id = server['id']
+                if server_id not in containers:
+                    containers[image_name][server_id] = server['name']
+
+print("Found following containers for selected images")
+#debug(containers, False)
+
+# Retrieve for each image type and containers
+metric_data = {}
+datapoints = 0
+for image in containers:
+    # Check if metric_data image exists
+    if image not in metric_data:
+        metric_data[image] = {}
+
+    for container_id in containers[image]:
+        metric_data[image][container_id] = {}
+
+    for n in range(int ((END_DATE - START_DATE).days)):
+        range_start_time = dt2ts(START_DATE + timedelta(n))
+        range_stop_time = dt2ts(START_DATE + timedelta(n + 1))
+
+        # Retrieve data for all containers for this timerange
+        servers = ','.join("s%s" % str(server_id) for server_id in containers[image])
+        values = API.get_data(METRIC_ID, servers, range_start_time, range_stop_time)
+
+        # Parse data for each container and add it
+        for value in values:
+            container_id = int(value['s'][2:-1]) # Remove the [s at beginning and ] at the end
+            metric_data[image][container_id][range_start_time] = {
+                'start': range_start_time,
+                'stop': range_stop_time,
+                'value': value['calc'][0],
+            }
+            datapoints = datapoints + 1
+
+print("Fetched %s datapoints" % datapoints)
+#debug(metric_data, False)
+
+# Retrieve all namespaces
+namespaces = {}
+
+# Retrieve Namespace group children
+namespace_group = API.get_servergroup('Namespaces')
+namespaces_group = API.get_servergroup_children(namespace_group['id'], start=START_TIME, stop=END_TIME)
+
+
+# Retrieve servers in namespaces
+for namespace in namespaces_group:
+    # Prepare data
+    namespace_name = namespace['name']
+    namespace_id = namespace['id']
+    if namespace_name not in namespaces:
+        namespaces[namespace_name] = {}
+
+    # Loop days to limit amount of servers we get back
+    for n in range(int ((END_DATE - START_DATE).days)):
+        range_start_time = dt2ts(START_DATE + timedelta(n))
+        range_stop_time = dt2ts(START_DATE + timedelta(n + 1))
+
+        serverIds = API.get_all_servers_in_group(namespace_id, range_start_time, range_stop_time)
+
+        # Check images
+        for image in metric_data:
+            if image not in namespaces[namespace_name]:
+                namespaces[namespace_name][image] = {}
+
+            for container in metric_data[image]:
+                total = 0.0
+                if container in serverIds:
+                    value = metric_data[image][container][range_start_time]['value']
+                    if value is not 'null' and value is not None:
+                        total += value
+
+                date = datetime.fromtimestamp(range_start_time).strftime('%Y-%m-%d')
+                if date not in namespaces[namespace_name][image]:
+                    namespaces[namespace_name][image][date] = 0
+
+                namespaces[namespace_name][image][date] += total
+
+debug(namespaces)