Merge branch 'main' into remove/cephScrapePublicDeployment

.gitignore

@@ -124,11 +124,6 @@ deployment_config.yaml
 /services/monitoring/grafana/provisioning/datasources/datasource.yml
 /services/monitoring/grafana/config.monitoring
 
-# Mail configuration
-/services/mail/config/*
-!/services/mail/config/amavis.cf
-services/monitoring/prometheus/prometheus.yml
-
 # Monitoring
 ### Grafana dashboard templated
 /services/monitoring/grafana/provisioning/dashboard/simcore/Metrics-dashboard.json

.pre-commit-config.yaml

@@ -1,10 +1,10 @@
 # See https://pre-commit.com/hooks.html for more hooks
 exclude: "^.venv$|^.cache$|^.pytest_cache$"
 default_language_version:
-  python: python3.9
+  python: python3.10
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.2.0
+    rev: v4.4.0
     hooks:
       - id: check-added-large-files
         args: ["--maxkb=499"] # ensures docker config compatability for all files
@@ -75,11 +75,9 @@ repos:
     rev: v2.34.0
     hooks:
       - id: pyupgrade
-        args:
-          - "--py39-plus"
         name: upgrade code
   - repo: https://github.com/hadialqattan/pycln
-    rev: v1.2.5
+    rev: v2.2.2
     hooks:
       - id: pycln
         args: [--all, --expand-stars]
@@ -91,7 +89,7 @@ repos:
         args: ["--profile", "black"]
         name: sort imports
   - repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.9.0
+    rev: v1.10.0
     hooks:
       - id: python-no-eval
         name: check for the `eval()` built-in python function
@@ -104,7 +102,6 @@ repos:
         (?x)^(
           services/monitoring/prometheus/.*\.rules\.yml
         )$
-
   #- repo: https://github.com/PyCQA/bandit
   #  rev: 1.7.4
   #  hooks:
@@ -116,7 +113,7 @@ repos:
       - id: shellcheck
         name: Shell scripts conform to shellcheck
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 23.7.0
     hooks:
       - id: black
         name: black format code

Makefile

@@ -27,7 +27,7 @@ certificates/domain.key:
 
 .PHONY: up-local
 up-local: .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up)
-	@bash scripts/deployments/deploy.sh --stack_target=local --minio_enabled=0 --disable_vcs_check=0
+	@bash scripts/deployments/deploy.sh --stack_target=local --minio_enabled=0 --vcs_check=1
 	@$(MAKE) info-local
 
 .PHONY: up-vagrant
@@ -45,15 +45,15 @@ up-simcore-dalco:  ## Deploy simcores stack only, on Dalco Cluster
 
 .PHONY: up-dalco
 up-dalco: ## Deploy ops and simcore stacks on the Dalco Cluster
-	./scripts/deployments/deploy.sh --stack_target=dalco
+	./scripts/deployments/deploy.sh --stack_target=dalco --vcs_check=1
 
 .PHONY: up-public
 up-public: ## Deploy ops and simcore stacks on the Public Cluster
-	./scripts/deployments/deploy.sh --stack_target=public
+	./scripts/deployments/deploy.sh --stack_target=public --vcs_check=1
 
 .PHONY: up-aws
 up-aws: ## Deploy opt and simcore stacks on the AWS Cluster
-	./scripts/deployments/deploy.sh --stack_target=aws
+	./scripts/deployments/deploy.sh --stack_target=aws --vcs_check=1
 
 .PHONY: up-master
 up-master: ## Deploy opt and simcore stacks on the Master Cluster

certificates/Makefile

@@ -38,7 +38,7 @@ rootca.crt: rootca.key
 		-sha384 -days 10000 -out $@;
 
 extfile.cnf:
-	@echo "subjectAltName = DNS:${MACHINE_FQDN}, DNS:testing.${MACHINE_FQDN}, DNS:services.testing.${MACHINE_FQDN}, DNS:*.services.testing.${MACHINE_FQDN}, DNS:services.${MACHINE_FQDN}, DNS:*.services.${MACHINE_FQDN}, DNS:${MAILSERVER_DOMAIN}, DNS:traefikdashboard.${MACHINE_FQDN}, DNS:invitations.${MACHINE_FQDN}, DNS:${MONITORING_DOMAIN}, DNS:${STORAGE_DOMAIN}, DNS:${FILESTASH_DOMAIN}, DNS:${REGISTRY_DOMAIN}, DNS:${API_DOMAIN}, DNS:testing.${API_DOMAIN}" > $@ #You can use IP:your_IP or DNS:host_name
+	@echo "subjectAltName = DNS:${MACHINE_FQDN}, DNS:testing.${MACHINE_FQDN}, DNS:services.testing.${MACHINE_FQDN}, DNS:*.services.testing.${MACHINE_FQDN}, DNS:services.${MACHINE_FQDN}, DNS:*.services.${MACHINE_FQDN}, DNS:traefikdashboard.${MACHINE_FQDN}, DNS:invitations.${MACHINE_FQDN}, DNS:${MONITORING_DOMAIN}, DNS:${STORAGE_DOMAIN}, DNS:${FILESTASH_DOMAIN}, DNS:${REGISTRY_DOMAIN}, DNS:${API_DOMAIN}, DNS:testing.${API_DOMAIN}" > $@ #You can use IP:your_IP or DNS:host_name
 
 domain.key:
 	# Creating private key

ci/ciRunsDockerContainer/Dockerfile

@@ -1,5 +1,6 @@
 ARG UBUNTU_VERSION="focal-20230412"
 FROM ubuntu:${UBUNTU_VERSION} as base
+ENV PYTHON_VERSION 3.10.12
 #
 LABEL maintainer=dkaiser
 
@@ -45,21 +46,55 @@ COPY mozillateam.pref /etc/apt/preferences.d/mozillateam.pref
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
   build-essential \
+  ca-certificates \
   curl \
-  firefox=114.0.2+build1-0ubuntu0.20.04.1 \
+  firefox=116.0.2+build1-0ubuntu0.20.04.1 \
   gettext-base \
   git \
   jq \
+  llvm \
   make \
   openssl \
   openssh-client \
   python3-pip \
   python3.9-venv \
+  python-is-python3 \
   unzip \
   wget \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
+# Install pyenv, modified from https://gist.github.com/jprjr/7667947?permalink_comment_id=3684823#gistcomment-3684823
+
+#Set of all dependencies needed for pyenv to work on Ubuntu
+RUN apt-get update \
+        && apt-get install -y --no-install-recommends \
+        libssl-dev \
+        zlib1g-dev \
+        libbz2-dev \
+        libreadline-dev \
+        libsqlite3-dev \
+        libncurses5-dev \
+        xz-utils \
+        tk-dev \
+        libxml2-dev \
+        libxmlsec1-dev \
+        libffi-dev \
+        liblzma-dev \
+        mecab-ipadic-utf8 git
+
+# Set-up necessary Env vars for PyEnv
+ENV PYENV_ROOT /root/.pyenv
+ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH
+
+# Install pyenv
+RUN set -ex \
+    && curl https://pyenv.run | bash \
+    && pyenv update \
+    && pyenv install $PYTHON_VERSION \
+    && pyenv global $PYTHON_VERSION \
+    && pyenv rehash
+
 # Install minio mc
 RUN echo "------------------------------------------------------ minio/mc" \
   && curl https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2022-10-01T07-56-14Z    --create-dirs   -o $HOME/minio-binaries/mc \
@@ -88,6 +123,17 @@ RUN echo "------------------------------------------------------ terraform" \
     && unzip terraform_1.5.1_linux_amd64.zip  -d /usr/local/bin \
     && rm /tmp/terraform_1.5.1_linux_amd64.zip
 
+# Install dind docker
+RUN mkdir -p /etc/apt/keyrings \
+  && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
+  && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+  $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends \
+  docker-ce-cli \
+  containerd.io \
+  docker-compose-plugin
+
 WORKDIR /home/scu
 
 CMD ["/bin/bash"]

ci/ciRunsDockerContainer/Makefile

@@ -1,42 +1,47 @@
-.DEFAULT_GOAL := help
 # Makefile including function used by every services
 include ../../scripts/common.Makefile
-include .env
 ## DOCKER TAGS  -------------------------------
 
 .PHONY: tag-local tag-version tag-latest build-container
 
 tag-version: .env ## Tags 'local/${DOCKER_IMAGE_NAME}):dev' images as versioned '${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}'
-	docker tag local/${DOCKER_IMAGE_NAME}:dev ${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG};
+	@set -o allexport; \
+	source $(realpath $(CURDIR)/.env); \
+	set +o allexport; \
+	docker tag local/$$DOCKER_IMAGE_NAME:dev $$DOCKER_REGISTRY/$$DOCKER_IMAGE_NAME:$$DOCKER_IMAGE_TAG;
 
 tag-latest: .env ## Tags last locally built production images as '${DOCKER_REGISTRY}/{service}:latest'
-	docker tag local/${DOCKER_IMAGE_NAME}:dev ${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:latest;
+	@set -o allexport; \
+	source $(realpath $(CURDIR)/.env); \
+	set +o allexport; \
+	docker tag local/$$DOCKER_IMAGE_NAME:dev $$DOCKER_REGISTRY/$$DOCKER_IMAGE_NAME:latest;
 
 build-container: .env ## Build the common docker container that runs the CI runs
-	docker build  --no-cache -t local/${DOCKER_IMAGE_NAME}:dev .;
-
+	@set -o allexport; \
+	source $(realpath $(CURDIR)/.env); \
+	set +o allexport; \
+	docker build  --no-cache -t local/$$DOCKER_IMAGE_NAME:dev .;
 
+.PHONY: build
+build: build-container ## Build the docker container that runs the CI runs
 ## DOCKER PULL/PUSH  -------------------------------
-#
-# TODO: do not allow pushing modified/untracked
-# TODO: do not allow pushing disceted
+
 #
 
 .PHONY: push-version push-latest
 
-push-latest: tag-latest
+push-latest: tag-latest ## Pushes '${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:latest' images to the registry
 	@set -o allexport; \
 	source $(realpath $(CURDIR)/.env); \
 	set +o allexport; \
 	export DOCKER_IMAGE_TAG=latest; \
-	$(MAKE) push-version
+	./dockerhub_login.bash && \
+	docker compose --file docker-compose-deploy.yml push
 
-# below BUILD_TARGET gets overwritten but is required when merging yaml files
-push-version: tag-version
-	# pushing '${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}'
+push-version: tag-version ## Pushes '${DOCKER_REGISTRY}/${DOCKER_IMAGE_NAME}:{DOCKER_IMAGE_TAG}' images to the registry
 	@set -o allexport; \
 	source $(realpath $(CURDIR)/.env); \
 	set +o allexport; \
-	export DOCKER_IMAGE_TAG=latest; \
+	echo "Pushing $$DOCKER_REGISTRY/$$DOCKER_IMAGE_NAME:$$DOCKER_IMAGE_TAG"; \
 	./dockerhub_login.bash && \
 	docker compose --file docker-compose-deploy.yml push

ci/ciRunsDockerContainer/template.env

@@ -1,5 +1,5 @@
 DOCKER_REGISTRY=itisfoundation
-DOCKER_IMAGE_TAG=0.2.1
+DOCKER_IMAGE_TAG=0.4.0
 DOCKER_IMAGE_NAME=ci-provisioned-ubuntu
 DOCKER_USERNAME=${DOCKER_HUB_LOGIN}
 DOCKER_PASSWORD=${DOCKER_HUB_PASSWORD}

scripts/common.Makefile

@@ -3,7 +3,6 @@ SHELL := /bin/bash
 MAKE_C := $(MAKE) --no-print-directory --directory
 PREDEFINED_VARIABLES := $(.VARIABLES)
 VERSION := $(shell uname -a)
-SWARM_HOSTS = $(shell docker node ls --format={{.Hostname}} 2>/dev/null)
 
 
 # Checks for handling various operating systems
@@ -208,10 +207,28 @@ clean-default: .check_clean ## Cleans all outputs
 # Helpers -------------------------------------------------
 .PHONY: .init
 .init: ## initializeds swarm cluster
-	@$(if $(SWARM_HOSTS),  \
-		,                 \
-		echo "SWARM IS NOT INITIALIZED. ABORTING! (Tip to solve this: Run `docker swarm init` and create a swarm) " && exit 1\
-	)
+	@node_state="$$(docker info --format '{{ .Swarm.LocalNodeState }}')"; \
+	if [ "$$node_state" = "inactive" ]; then \
+		echo "This node is not in a swarm cluster. In production, use ansible to properly initialize a swarm cluster."; \
+		read -p "Do you want to initialize a swarm cluster? (y/n): " answer; \
+		if [ "$$answer" = "n" ]; then \
+			echo "Swarm initialization canceled. Script cannot proceed without an initialized swarm cluster"; \
+			exit 1; \
+		fi; \
+		echo "Initializing a swarm cluster"; \
+		docker swarm init > /dev/null; \
+		docker node ls --quiet | xargs -I {} sh -c ' \
+			docker node update --label-add simcore=true \
+				--label-add dynamicsidecar=true \
+				--label-add dasksidecar=true \
+				--label-add rabbit=true \
+				--label-add redis=true \
+				--label-add traefik=true \
+				--label-add ops=true \
+				--label-add prometheus=true \
+			 	--label-add minio=true {}' > /dev/null; \
+	fi
+
 	@$(if $(filter $(PUBLIC_NETWORK), $(shell docker network ls --format="{{.Name}}")) \
 		,  \
 		, docker network create --attachable --driver=overlay --subnet=10.10.0.0/16 $(PUBLIC_NETWORK) \