✨ (⚠️ devops) 🗃️ Is922 resource tracking/1. version of regular scraping

packages/postgres-database/src/simcore_postgres_database/migration/versions/967cff1d52ac_new_resource_tracker_container_table.py

@@ -0,0 +1,49 @@
+"""New resource_tracker_container table
+
+Revision ID: 967cff1d52ac
+Revises: c8f072c72adc
+Create Date: 2023-06-13 06:37:31.542190+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "967cff1d52ac"
+down_revision = "c8f072c72adc"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "resource_tracker_container",
+        sa.Column("id", sa.String(), nullable=False),
+        sa.Column("image", sa.String(), nullable=False),
+        sa.Column("user_id", sa.BigInteger, nullable=False),
+        sa.Column("product_name", sa.String(), nullable=False),
+        sa.Column("cpu_reservation", sa.BigInteger(), nullable=False),
+        sa.Column("ram_reservation", sa.BigInteger(), nullable=False),
+        sa.Column("container_cpu_usage_seconds_total", sa.Float(), nullable=False),
+        sa.Column("created_timestamp", sa.DateTime(timezone=True), nullable=False),
+        sa.Column(
+            "last_prometheus_scraped_timestamp",
+            sa.DateTime(timezone=True),
+            nullable=False,
+        ),
+        sa.Column(
+            "last_row_updated_timestamp",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=False,
+        ),
+        sa.PrimaryKeyConstraint("id", name="resource_tracker_container_pkey"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("resource_tracker_container")
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/resource_tracker.py

@@ -0,0 +1,60 @@
+""" resource_tracker_container table
+
+    - Table where we store the resource usage of each container that
+    we scrape via resource-usage-tracker service
+"""
+
+import sqlalchemy as sa
+from sqlalchemy.sql import func
+
+from .base import metadata
+
+resource_tracker_container = sa.Table(
+    "resource_tracker_container",
+    metadata,
+    sa.Column(
+        "id",  # container_id
+        sa.String,
+        nullable=False,
+    ),
+    sa.Column(
+        "image",
+        sa.String,
+        nullable=False,
+        doc="ex. registry.osparc.io/simcore/services/dynamic/jupyter-smash:3.0.9",
+    ),
+    sa.Column(
+        "user_id",
+        sa.BigInteger,
+        nullable=False,
+    ),
+    sa.Column(
+        "product_name",
+        sa.String,
+        nullable=False,
+    ),
+    sa.Column("cpu_reservation", sa.BigInteger, nullable=False),
+    sa.Column("ram_reservation", sa.BigInteger, nullable=False),
+    sa.Column("container_cpu_usage_seconds_total", sa.Float, nullable=False),
+    sa.Column(
+        "created_timestamp",
+        sa.DateTime(timezone=True),
+        nullable=False,
+        doc="First container creation timestamp (UTC timestamp)",
+    ),
+    sa.Column(
+        "last_prometheus_scraped_timestamp",
+        sa.DateTime(timezone=True),
+        nullable=False,
+        doc="Last prometheus scraped timestamp (UTC timestamp)",
+    ),
+    sa.Column(
+        "last_row_updated_timestamp",
+        sa.DateTime(timezone=True),
+        nullable=False,
+        server_default=func.now(),
+        doc="Last row updated timestamp (UTC timestamp)",
+    ),
+    # ---------------------------
+    sa.PrimaryKeyConstraint("id", name="resource_tracker_container_pkey"),
+)

services/docker-compose.devel.yml

@@ -124,6 +124,16 @@ services:
       - SC_BOOT_MODE=debug-ptvsd
       - RESOURCE_USAGE_TRACKER_LOGLEVEL=DEBUG
       - DEBUG=true
+      - LOG_FORMAT_LOCAL_DEV_ENABLED=${LOG_FORMAT_LOCAL_DEV_ENABLED}
+      - POSTGRES_DB=${POSTGRES_DB}
+      - POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT}
+      - POSTGRES_HOST=${POSTGRES_HOST}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_PORT=${POSTGRES_PORT}
+      - POSTGRES_USER=${POSTGRES_USER}
+      - PROMETHEUS_URL=${RESOURCE_USAGE_TRACKER_PROMETHEUS_URL}
+      - PROMETHEUS_USERNAME=${RESOURCE_USAGE_TRACKER_PROMETHEUS_USERNAME}
+      - PROMETHEUS_PASSWORD=${RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD}
     volumes:
       - ./resource-usage-tracker:/devel/services/resource-usage-tracker
       - ../packages:/devel/packages

services/docker-compose.yml

@@ -194,9 +194,9 @@ services:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_PORT=${POSTGRES_PORT}
       - POSTGRES_USER=${POSTGRES_USER}
-      - PROMETHEUS_URL=${PROMETHEUS_URL}
-      - PROMETHEUS_USERNAME=${PROMETHEUS_USERNAME}
-      - PROMETHEUS_PASSWORD=${PROMETHEUS_PASSWORD}
+      - PROMETHEUS_URL=${RESOURCE_USAGE_TRACKER_PROMETHEUS_URL}
+      - PROMETHEUS_USERNAME=${RESOURCE_USAGE_TRACKER_PROMETHEUS_USERNAME}
+      - PROMETHEUS_PASSWORD=${RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD}
       - RESOURCE_USAGE_TRACKER_LOGLEVEL=${LOG_LEVEL:-INFO}
 
   static-webserver:

services/resource-usage-tracker/.env-devel

@@ -0,0 +1,8 @@
+RESOURCE_USAGE_TRACKER_DEV_FEATURES_ENABLED=1
+
+LOG_LEVEL=DEBUG
+
+POSTGRES_USER=test
+POSTGRES_PASSWORD=test
+POSTGRES_DB=test
+POSTGRES_HOST=localhost

services/resource-usage-tracker/requirements/_base.in

@@ -8,13 +8,17 @@
 # intra-repo required dependencies
 --requirement ../../../packages/models-library/requirements/_base.in
 --requirement ../../../packages/settings-library/requirements/_base.in
+--requirement ../../../packages/postgres-database/requirements/_base.in
 # service-library[fastapi]
 --requirement ../../../packages/service-library/requirements/_base.in
 --requirement ../../../packages/service-library/requirements/_fastapi.in
 
 
+asyncpg # database
 fastapi
 packaging
 prometheus_api_client
 typer[all]
 uvicorn[standard]
+aiopg[sa]
+orjson