From 9f6094891b441e55a79f1e613050aba627d6086c Mon Sep 17 00:00:00 2001 From: sanderegg <35365065+sanderegg@users.noreply.github.com> Date: Tue, 4 Jul 2023 15:03:03 +0200 Subject: [PATCH] new docker label dictionary --- .../src/models_library/docker.py | 127 +++++++++++++++--- packages/models-library/tests/test_docker.py | 44 ++---- 2 files changed, 118 insertions(+), 53 deletions(-) diff --git a/packages/models-library/src/models_library/docker.py b/packages/models-library/src/models_library/docker.py index 8ffbbd1e958a..829bf7afa2af 100644 --- a/packages/models-library/src/models_library/docker.py +++ b/packages/models-library/src/models_library/docker.py @@ -1,3 +1,4 @@ +import contextlib import re from typing import Any, Final @@ -6,9 +7,18 @@ from models_library.projects import ProjectID from models_library.projects_nodes import NodeID from models_library.users import UserID -from pydantic import BaseModel, ByteSize, ConstrainedStr, Field, root_validator +from pydantic import ( + BaseModel, + ByteSize, + ConstrainedStr, + Field, + ValidationError, + parse_obj_as, + root_validator, +) from .basic_regex import DOCKER_GENERIC_TAG_KEY_RE, DOCKER_LABEL_KEY_REGEX +from .utils.fastapi_encoders import jsonable_encoder class DockerLabelKey(ConstrainedStr): @@ -24,14 +34,18 @@ class DockerGenericTag(ConstrainedStr): _SIMCORE_CONTAINER_PREFIX: Final[str] = "io.simcore.container." _BACKWARDS_COMPATIBILITY_MAP: Final[dict[str, str]] = { - "user_id": f"{_SIMCORE_CONTAINER_PREFIX}user-id", - "study_id": f"{_SIMCORE_CONTAINER_PREFIX}project-id", - "project_id": f"{_SIMCORE_CONTAINER_PREFIX}project-id", - "uuid": f"{_SIMCORE_CONTAINER_PREFIX}node-id", "node_id": f"{_SIMCORE_CONTAINER_PREFIX}node-id", "product_name": f"{_SIMCORE_CONTAINER_PREFIX}product-name", + "project_id": f"{_SIMCORE_CONTAINER_PREFIX}project-id", "simcore_user_agent": f"{_SIMCORE_CONTAINER_PREFIX}simcore-user-agent", + "study_id": f"{_SIMCORE_CONTAINER_PREFIX}project-id", + "user_id": f"{_SIMCORE_CONTAINER_PREFIX}user-id", + "uuid": f"{_SIMCORE_CONTAINER_PREFIX}node-id", + "mem_limit": f"{_SIMCORE_CONTAINER_PREFIX}memory-limit", + "swarm_stack_name": f"{_SIMCORE_CONTAINER_PREFIX}swarm-stack-name", } +_UNDEFINED_VALUE_STR: Final[str] = "undefined" +_UNDEFINED_VALUE_INT: Final[str] = "0" class SimcoreServiceDockerLabelKeys(BaseModel): @@ -46,36 +60,52 @@ class SimcoreServiceDockerLabelKeys(BaseModel): ..., alias=f"{_SIMCORE_CONTAINER_PREFIX}simcore-user-agent" ) - # None is for backwards compatibility, can be removed in a few sprints - memory_limit: ByteSize | None = Field( + swarm_stack_name: str = Field( + ..., alias=f"{_SIMCORE_CONTAINER_PREFIX}swarm-stack-name" + ) + + memory_limit: ByteSize = Field( ..., alias=f"{_SIMCORE_CONTAINER_PREFIX}memory-limit" ) - cpu_limit: float | None = Field(..., alias=f"{_SIMCORE_CONTAINER_PREFIX}cpu-limit") + cpu_limit: float = Field(..., alias=f"{_SIMCORE_CONTAINER_PREFIX}cpu-limit") @root_validator(pre=True) def _backwards_compatibility(cls, values: dict[str, Any]) -> dict[str, Any]: - # NOTE: this is necessary for deployment and legacy service + # NOTE: this is necessary for dy-sidecar and legacy service until they are adjusted if mapped_values := { _BACKWARDS_COMPATIBILITY_MAP[k]: v for k, v in values.items() if k in _BACKWARDS_COMPATIBILITY_MAP }: + # these values were sometimes omitted, so let's provide some defaults + for key in ["product-name", "simcore-user-agent", "swarm-stack-name"]: + mapped_values.setdefault( + f"{_SIMCORE_CONTAINER_PREFIX}{key}", _UNDEFINED_VALUE_STR + ) + mapped_values.setdefault( - f"{_SIMCORE_CONTAINER_PREFIX}product-name", "osparc" + f"{_SIMCORE_CONTAINER_PREFIX}memory-limit", _UNDEFINED_VALUE_INT ) + + def _convert_nano_cpus_to_cpus(nano_cpu: str) -> str: + with contextlib.suppress(ValidationError): + return f"{parse_obj_as(float, nano_cpu) / (1.0*10**9):.2f}" + return _UNDEFINED_VALUE_INT + mapped_values.setdefault( - f"{_SIMCORE_CONTAINER_PREFIX}simcore-user-agent", "undefined" + f"{_SIMCORE_CONTAINER_PREFIX}cpu-limit", + _convert_nano_cpus_to_cpus( + values.get("nano_cpus_limit", _UNDEFINED_VALUE_INT) + ), ) - mapped_values.setdefault(f"{_SIMCORE_CONTAINER_PREFIX}memory-limit", "0") - mapped_values.setdefault(f"{_SIMCORE_CONTAINER_PREFIX}cpu-limit", "0") return mapped_values return values def to_docker_labels(self) -> dict[DockerLabelKey, str]: """returns a dictionary of strings as required by docker""" - std_export = self.dict(by_alias=True) + std_export = jsonable_encoder(self, by_alias=True) return { - DockerLabelKey(f"{_SIMCORE_CONTAINER_PREFIX}{k.replace('_', '-')}"): f"{v}" + DockerLabelKey(f"{k.replace('_', '-')}"): f"{v}" for k, v in sorted(std_export.items()) } @@ -90,23 +120,80 @@ class Config: allow_population_by_field_name = True schema_extra = { "examples": [ - # legacy with no limits (a.k.a all dynamic services) + # legacy service labels { + "port": "8080", # TODO: go away + "study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6", + "swarm_stack_name": "devel-simcore", + "type": "main", # TODO: go away + "user_id": "5", "uuid": "1f963626-66e1-43f1-a777-33955c08b909", + }, + # legacy container labels + { + "mem_limit": "1073741824", + "nano_cpus_limit": "4000000000", + "node_id": "1f963626-66e1-43f1-a777-33955c08b909", + "simcore_user_agent": "puppeteer", "study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6", + "swarm_stack_name": "devel-simcore", "user_id": "5", + }, + # dy-sidecar service labels + { + "key": "simcore/services/dynamic/jupyter-math", # TODO: go away + "port": "8888", # TODO: go away + "service_image": "itisfoundation/dynamic-sidecar:master-github-latest", # TODO: go away + "service_port": "8888", # TODO: go away + "study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6", + "swarm_stack_name": "devel-simcore", + "type": "main-v2", # TODO: go away + "user_id": "5", + "uuid": "1f963626-66e1-43f1-a777-33955c08b909", + "version": "2.0.9", # TODO: go away + # "simcore_user_agent": "???????misssing??" + }, + # dy-sidecar container labels + { + "mem_limit": "1073741824", + "nano_cpus_limit": "4000000000", + "study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6", + "user_id": "5", + "uuid": "1f963626-66e1-43f1-a777-33955c08b909", + }, + # dy-proxy service labels + { + "dynamic-type": "dynamic-sidecar", + "study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6", + "swarm_stack_name": "devel-simcore", + "type": "dependency-v2", + "user_id": "5", + "uuid": "1f963626-66e1-43f1-a777-33955c08b909", + }, + # dy-proxy container labels + { + "study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6", + "user_id": "5", + "uuid": "1f963626-66e1-43f1-a777-33955c08b909", + }, + # dy-sidecar user-services labels + { "product_name": "osparc", "simcore_user_agent": "puppeteer", + "study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6", + "user_id": "5", + "uuid": "1f963626-66e1-43f1-a777-33955c08b909", }, # modern both dynamic-sidecar services and computational services { + "io.simcore.container.cpu-limit": "2.4", + "io.simcore.container.memory-limit": "1073741824", "io.simcore.container.node-id": "1f963626-66e1-43f1-a777-33955c08b909", - "io.simcore.container.project-id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6", - "io.simcore.container.user-id": "5", "io.simcore.container.product-name": "osparc", + "io.simcore.container.project-id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6", "io.simcore.container.simcore-user-agent": "puppeteer", - "io.simcore.container.memory-limit": "1073741824", - "io.simcore.container.cpu-limit": "2.4", + "io.simcore.container.swarm-stack-name": "devel-osparc", + "io.simcore.container.user-id": "5", }, ] } diff --git a/packages/models-library/tests/test_docker.py b/packages/models-library/tests/test_docker.py index 8dc5ead5fb7a..97e833a0a485 100644 --- a/packages/models-library/tests/test_docker.py +++ b/packages/models-library/tests/test_docker.py @@ -8,6 +8,7 @@ import pytest from faker import Faker from models_library.docker import ( + _SIMCORE_CONTAINER_PREFIX, DockerGenericTag, DockerLabelKey, SimcoreServiceDockerLabelKeys, @@ -106,42 +107,19 @@ def test_docker_generic_tag(image_name: str, valid: bool): @pytest.mark.parametrize( - "obj_data", - [ - pytest.param( - { - "user_id": _faker.pyint(), - "project_id": _faker.uuid4(), - "node_id": _faker.uuid4(), - }, - id="parse_existing_service_labels-and-legacy", - ), - pytest.param( - { - "user_id": _faker.pyint(), - "project_id": _faker.uuid4(), - "node_id": _faker.uuid4(), - "product_name": "test_p", - "simcore_user_agent": "a-test-puppet", - }, - id="parse_new_service_labels-soon-to-removed", - ), - pytest.param( - { - "io.simcore.container.user_id": _faker.pyint(), - "io.simcore.container.project_id": _faker.uuid4(), - "io.simcore.container.node_id": _faker.uuid4(), - "io.simcore.container.product-name": "test_p", - "io.simcore.container.simcore_user_agent": "a-test-puppet", - }, - id="labels following docker rules", - ), - ], + "obj_data", SimcoreServiceDockerLabelKeys.Config.schema_extra["examples"], ids=str ) def test_simcore_service_docker_label_keys(obj_data: dict[str, Any]): simcore_service_docker_label_keys = SimcoreServiceDockerLabelKeys.parse_obj( obj_data ) exported_dict = simcore_service_docker_label_keys.to_docker_labels() - assert all(isinstance(v, str) for v in exported_dict.values()) - assert parse_obj_as(SimcoreServiceDockerLabelKeys, exported_dict) + assert all( + isinstance(v, str) for v in exported_dict.values() + ), "docker labels must be strings!" + assert all(key.startswith(_SIMCORE_CONTAINER_PREFIX) for key in exported_dict) + re_imported_docker_label_keys = parse_obj_as( + SimcoreServiceDockerLabelKeys, exported_dict + ) + assert re_imported_docker_label_keys + assert simcore_service_docker_label_keys == re_imported_docker_label_keys