Add an ability to specify registry when using docker images (#2008)

mathbunnyru · web-flow · commit f8cd90ade13a · 2023-10-19T21:15:10.000+02:00
* Add an ability to specify registry when using docker images * Fix typo * [TMP] Speedup workflow * Revert "[TMP] Speedup workflow" This reverts commit 3af0055.
diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml
@@ -1,6 +1,7 @@
 name: Download parent image, build a new one and test it; then upload the image, tags and manifests to GitHub artifacts
 
 env:
+  REGISTRY: docker.io
   OWNER: ${{ github.repository_owner }}
 
 on:
@@ -60,20 +61,20 @@ jobs:
         shell: bash
 
       - name: Build image 🛠
-        run: docker build --rm --force-rm --tag ${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/ --build-arg OWNER=${{ env.OWNER }}
+        run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }}
         env:
           DOCKER_BUILDKIT: 1
           # Full logs for CI build
           BUILDKIT_PROGRESS: plain
         shell: bash
 
       - name: Run tests ✅
-        run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --owner ${{ env.OWNER }}
+        run: python3 -m tests.run_tests --short-image-name ${{ inputs.image }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
         shell: bash
 
       - name: Write tags file 🏷
         run: |
-          python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --owner ${{ env.OWNER }}
+          python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
         shell: bash
       - name: Upload tags file 💾
         uses: actions/upload-artifact@v3
@@ -83,7 +84,7 @@ jobs:
           retention-days: 3
 
       - name: Write manifest and build history file 🏷
-        run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --owner ${{ env.OWNER }}
+        run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
         shell: bash
       - name: Upload manifest file 💾
         uses: actions/upload-artifact@v3
diff --git a/.github/workflows/docker-tag-push.yml b/.github/workflows/docker-tag-push.yml
@@ -1,6 +1,7 @@
 name: Download Docker image and its tags from GitHub artifacts, apply them and push the image to Docker Hub
 
 env:
+  REGISTRY: docker.io
   OWNER: ${{ github.repository_owner }}
 
 on:
@@ -50,7 +51,7 @@ jobs:
           name: ${{ inputs.image }}-${{ inputs.platform }}-tags
           path: /tmp/jupyter/tags/
       - name: Apply tags to the loaded image 🏷
-        run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --owner ${{ env.OWNER }}
+        run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
 
       - name: Push Images to Docker Hub 📤
         if: github.repository == 'jupyter/docker-stacks' && (github.ref == 'refs/heads/main' || github.event_name == 'schedule')
diff --git a/Makefile b/Makefile
@@ -4,6 +4,7 @@
 
 # Use bash for inline if-statements in arch_patch target
 SHELL:=bash
+REGISTRY?=docker.io
 OWNER?=jupyter
 
 # Need to list the images in build dependency order
@@ -37,9 +38,9 @@ help:
 
 build/%: DOCKER_BUILD_ARGS?=
 build/%: ## build the latest image for a stack using the system's architecture
-	docker build $(DOCKER_BUILD_ARGS) --rm --force-rm --tag "$(OWNER)/$(notdir $@):latest" "./images/$(notdir $@)" --build-arg OWNER="$(OWNER)"
+	docker build $(DOCKER_BUILD_ARGS) --rm --force-rm --tag "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" "./images/$(notdir $@)" --build-arg REGISTRY="$(REGISTRY)" --build-arg OWNER="$(OWNER)"
 	@echo -n "Built image size: "
-	@docker images "$(OWNER)/$(notdir $@):latest" --format "{{.Size}}"
+	@docker images "$(REGISTRY)/$(OWNER)/$(notdir $@):latest" --format "{{.Size}}"
 build-all: $(foreach I, $(ALL_IMAGES), build/$(I)) ## build all stacks
 
 
@@ -68,9 +69,9 @@ linkcheck-docs: ## check broken links
 
 
 hook/%: ## run post-build hooks for an image
-	python3 -m tagging.write_tags_file --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --owner "$(OWNER)" && \
-	python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --owner "$(OWNER)" && \
-	python3 -m tagging.apply_tags --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --platform "$(shell uname -m)" --owner "$(OWNER)"
+	python3 -m tagging.write_tags_file --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --registry "$(REGISTRY)" --owner "$(OWNER)" && \
+	python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --registry "$(REGISTRY)" --owner "$(OWNER)" && \
+	python3 -m tagging.apply_tags --short-image-name "$(notdir $@)" --tags-dir /tmp/jupyter/tags/ --platform "$(shell uname -m)" --registry "$(REGISTRY)" --owner "$(OWNER)"
 hook-all: $(foreach I, $(ALL_IMAGES), hook/$(I)) ## run post-build hooks for all images
 
 
@@ -105,5 +106,5 @@ run-sudo-shell/%: ## run a bash in interactive mode as root in a stack
 
 
 test/%: ## run tests against a stack
-	python3 -m tests.run_tests --short-image-name "$(notdir $@)" --owner "$(OWNER)"
+	python3 -m tests.run_tests --short-image-name "$(notdir $@)" --registry "$(REGISTRY)" --owner "$(OWNER)"
 test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks
diff --git a/binder/Dockerfile b/binder/Dockerfile
@@ -2,8 +2,9 @@
 # Distributed under the terms of the Modified BSD License.
 
 # https://hub.docker.com/r/jupyter/base-notebook/tags
+ARG REGISTRY=docker.io
 ARG OWNER=jupyter
-ARG BASE_CONTAINER=$OWNER/base-notebook:2023-09-25
+ARG BASE_CONTAINER=$REGISTRY/$OWNER/base-notebook:2023-09-25
 FROM $BASE_CONTAINER
 
 LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
diff --git a/docs/using/recipe_code/custom_environment.dockerfile b/docs/using/recipe_code/custom_environment.dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook
+FROM docker.io/jupyter/base-notebook
 
 # Name your environment and choose the python version
 ARG env_name=python310
diff --git a/docs/using/recipe_code/dask_jupyterlab.dockerfile b/docs/using/recipe_code/dask_jupyterlab.dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook
+FROM docker.io/jupyter/base-notebook
 
 # Install the Dask dashboard
 RUN mamba install --yes 'dask-labextension' && \
diff --git a/docs/using/recipe_code/jupyterhub_version.dockerfile b/docs/using/recipe_code/jupyterhub_version.dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook
+FROM docker.io/jupyter/base-notebook
 
 RUN mamba install --yes 'jupyterhub==4.0.1' && \
     mamba clean --all -f -y && \
diff --git a/docs/using/recipe_code/mamba_install.dockerfile b/docs/using/recipe_code/mamba_install.dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook
+FROM docker.io/jupyter/base-notebook
 
 RUN mamba install --yes 'flake8' && \
     mamba clean --all -f -y && \
diff --git a/docs/using/recipe_code/manpage_install.dockerfile b/docs/using/recipe_code/manpage_install.dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook
+FROM docker.io/jupyter/base-notebook
 
 # Fix: https://github.com/hadolint/hadolint/wiki/DL4006
 # Fix: https://github.com/koalaman/shellcheck/wiki/SC3014
diff --git a/docs/using/recipe_code/microsoft_odbc.dockerfile b/docs/using/recipe_code/microsoft_odbc.dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook
+FROM docker.io/jupyter/base-notebook
 
 # Fix: https://github.com/hadolint/hadolint/wiki/DL4006
 # Fix: https://github.com/koalaman/shellcheck/wiki/SC3014
diff --git a/docs/using/recipe_code/oracledb.dockerfile b/docs/using/recipe_code/oracledb.dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook
+FROM docker.io/jupyter/base-notebook
 
 USER root
 
diff --git a/docs/using/recipe_code/pip_install.dockerfile b/docs/using/recipe_code/pip_install.dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook
+FROM docker.io/jupyter/base-notebook
 
 # Install in the default python3 environment
 RUN pip install --no-cache-dir 'flake8' && \
diff --git a/docs/using/recipe_code/rise_jupyterlab.dockerfile b/docs/using/recipe_code/rise_jupyterlab.dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook
+FROM docker.io/jupyter/base-notebook
 
 RUN mamba install --yes 'jupyterlab_rise' && \
     mamba clean --all -f -y && \
diff --git a/docs/using/recipe_code/spellcheck_notebookv6.dockerfile b/docs/using/recipe_code/spellcheck_notebookv6.dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook:notebook-6.5.4
+FROM docker.io/jupyter/base-notebook:notebook-6.5.4
 
 RUN pip install --no-cache-dir 'jupyter_contrib_nbextensions' && \
     jupyter contrib nbextension install --user && \
diff --git a/docs/using/recipe_code/xgboost.dockerfile b/docs/using/recipe_code/xgboost.dockerfile
@@ -1,4 +1,4 @@
-FROM jupyter/base-notebook
+FROM docker.io/jupyter/base-notebook
 
 RUN mamba install --yes 'py-xgboost' && \
     mamba clean --all -f -y && \
diff --git a/docs/using/recipes.md b/docs/using/recipes.md
@@ -298,7 +298,7 @@ This recipe is not tested and might be broken.
 ```
 
 ```dockerfile
-FROM jupyter/all-spark-notebook
+FROM docker.io/jupyter/all-spark-notebook
 
 # Set env vars for pydoop
 ENV HADOOP_HOME /usr/local/hadoop-2.7.3
@@ -415,7 +415,7 @@ Please note that the [Delta Lake](https://delta.io/) packages are only available
 By adding the properties to `spark-defaults.conf`, the user no longer needs to enable Delta support in each notebook.
 
 ```dockerfile
-FROM jupyter/pyspark-notebook
+FROM docker.io/jupyter/pyspark-notebook
 
 RUN mamba install --yes 'delta-spark' && \
     mamba clean --all -f -y && \
@@ -446,7 +446,7 @@ This recipe is not tested and might be broken.
 The example below is a Dockerfile to load Source Han Sans with normal weight, usually used for the web.
 
 ```dockerfile
-FROM jupyter/scipy-notebook
+FROM docker.io/jupyter/scipy-notebook
 
 RUN PYV=$(ls "${CONDA_DIR}/lib" | grep ^python) && \
     MPL_DATA="${CONDA_DIR}/lib/${PYV}/site-packages/matplotlib/mpl-data" && \
@@ -489,7 +489,7 @@ This recipe is not tested and might be broken.
 The example below is a Dockerfile to install the [ijavascript kernel](https://github.com/n-riesco/ijavascript).
 
 ```dockerfile
-FROM jupyter/scipy-notebook
+FROM docker.io/jupyter/scipy-notebook
 
 # install ijavascript
 RUN npm install -g ijavascript
diff --git a/examples/docker-compose/README.md b/examples/docker-compose/README.md
@@ -42,7 +42,7 @@ You can customize the docker-stack notebook image to deploy by modifying the `no
 For example, you can build and deploy a `jupyter/all-spark-notebook` by modifying the Dockerfile like so:
 
 ```dockerfile
-FROM jupyter/all-spark-notebook
+FROM docker.io/jupyter/all-spark-notebook
 # Your RUN commands and so on
 ```
 
diff --git a/examples/docker-compose/notebook/Dockerfile b/examples/docker-compose/notebook/Dockerfile
@@ -2,7 +2,7 @@
 # Distributed under the terms of the Modified BSD License.
 
 # Pick your favorite docker-stacks image
-FROM jupyter/minimal-notebook
+FROM docker.io/jupyter/minimal-notebook
 
 USER root
 
diff --git a/examples/make-deploy/Dockerfile b/examples/make-deploy/Dockerfile
@@ -2,7 +2,7 @@
 # Distributed under the terms of the Modified BSD License.
 
 # Pick your favorite docker-stacks image
-FROM jupyter/minimal-notebook
+FROM docker.io/jupyter/minimal-notebook
 
 USER root
 
diff --git a/images/all-spark-notebook/Dockerfile b/images/all-spark-notebook/Dockerfile
@@ -1,7 +1,8 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+ARG REGISTRY=docker.io
 ARG OWNER=jupyter
-ARG BASE_CONTAINER=$OWNER/pyspark-notebook
+ARG BASE_CONTAINER=$REGISTRY/$OWNER/pyspark-notebook
 FROM $BASE_CONTAINER
 
 LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
diff --git a/images/base-notebook/Dockerfile b/images/base-notebook/Dockerfile
@@ -1,7 +1,8 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+ARG REGISTRY=docker.io
 ARG OWNER=jupyter
-ARG BASE_CONTAINER=$OWNER/docker-stacks-foundation
+ARG BASE_CONTAINER=$REGISTRY/$OWNER/docker-stacks-foundation
 FROM $BASE_CONTAINER
 
 LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
diff --git a/images/datascience-notebook/Dockerfile b/images/datascience-notebook/Dockerfile
@@ -1,7 +1,8 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+ARG REGISTRY=docker.io
 ARG OWNER=jupyter
-ARG BASE_CONTAINER=$OWNER/scipy-notebook
+ARG BASE_CONTAINER=$REGISTRY/$OWNER/scipy-notebook
 FROM $BASE_CONTAINER
 
 LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
diff --git a/images/julia-notebook/Dockerfile b/images/julia-notebook/Dockerfile
@@ -1,7 +1,8 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+ARG REGISTRY=docker.io
 ARG OWNER=jupyter
-ARG BASE_CONTAINER=$OWNER/minimal-notebook
+ARG BASE_CONTAINER=$REGISTRY/$OWNER/minimal-notebook
 FROM $BASE_CONTAINER
 
 LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
diff --git a/images/minimal-notebook/Dockerfile b/images/minimal-notebook/Dockerfile
@@ -1,7 +1,8 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+ARG REGISTRY=docker.io
 ARG OWNER=jupyter
-ARG BASE_CONTAINER=$OWNER/base-notebook
+ARG BASE_CONTAINER=$REGISTRY/$OWNER/base-notebook
 FROM $BASE_CONTAINER
 
 LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
diff --git a/images/pyspark-notebook/Dockerfile b/images/pyspark-notebook/Dockerfile
@@ -1,7 +1,8 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+ARG REGISTRY=docker.io
 ARG OWNER=jupyter
-ARG BASE_CONTAINER=$OWNER/scipy-notebook
+ARG BASE_CONTAINER=$REGISTRY/$OWNER/scipy-notebook
 FROM $BASE_CONTAINER
 
 LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
diff --git a/images/r-notebook/Dockerfile b/images/r-notebook/Dockerfile
@@ -1,7 +1,8 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+ARG REGISTRY=docker.io
 ARG OWNER=jupyter
-ARG BASE_CONTAINER=$OWNER/minimal-notebook
+ARG BASE_CONTAINER=$REGISTRY/$OWNER/minimal-notebook
 FROM $BASE_CONTAINER
 
 LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
diff --git a/images/scipy-notebook/Dockerfile b/images/scipy-notebook/Dockerfile
@@ -1,7 +1,8 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+ARG REGISTRY=docker.io
 ARG OWNER=jupyter
-ARG BASE_CONTAINER=$OWNER/minimal-notebook
+ARG BASE_CONTAINER=$REGISTRY/$OWNER/minimal-notebook
 FROM $BASE_CONTAINER
 
 LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
diff --git a/images/tensorflow-notebook/Dockerfile b/images/tensorflow-notebook/Dockerfile
@@ -1,7 +1,8 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+ARG REGISTRY=docker.io
 ARG OWNER=jupyter
-ARG BASE_CONTAINER=$OWNER/scipy-notebook
+ARG BASE_CONTAINER=$REGISTRY/$OWNER/scipy-notebook
 FROM $BASE_CONTAINER
 
 LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
diff --git a/tagging/apply_tags.py b/tagging/apply_tags.py
@@ -16,6 +16,7 @@
 
 def apply_tags(
     short_image_name: str,
+    registry: str,
     owner: str,
     tags_dir: Path,
     platform: str,
@@ -26,7 +27,7 @@ def apply_tags(
     """
     LOGGER.info(f"Tagging image: {short_image_name}")
 
-    image = f"{owner}/{short_image_name}:latest"
+    image = f"{registry}/{owner}/{short_image_name}:latest"
     filename = f"{platform}-{short_image_name}.txt"
     tags = (tags_dir / filename).read_text().splitlines()
 
@@ -60,6 +61,13 @@ def apply_tags(
         choices=["x86_64", "aarch64", "arm64"],
         help="Image platform",
     )
+    arg_parser.add_argument(
+        "--registry",
+        required=True,
+        type=str,
+        choices=["docker.io", "quay.io"],
+        help="Image registry",
+    )
     arg_parser.add_argument(
         "--owner",
         required=True,
@@ -68,4 +76,6 @@ def apply_tags(
     args = arg_parser.parse_args()
     args.platform = unify_aarch64(args.platform)
 
-    apply_tags(args.short_image_name, args.owner, args.tags_dir, args.platform)
+    apply_tags(
+        args.short_image_name, args.registry, args.owner, args.tags_dir, args.platform
+    )
diff --git a/tagging/manifests.py b/tagging/manifests.py
@@ -23,13 +23,18 @@ class ManifestHeader:
     """ManifestHeader doesn't fall under common interface, and we run it separately"""
 
     @staticmethod
-    def create_header(short_image_name: str, owner: str, build_timestamp: str) -> str:
+    def create_header(
+        short_image_name: str, registry: str, owner: str, build_timestamp: str
+    ) -> str:
         commit_hash = GitHelper.commit_hash()
         commit_hash_tag = GitHelper.commit_hash_tag()
         commit_message = GitHelper.commit_message()
 
         image_size = docker[
-            "images", f"{owner}/{short_image_name}:latest", "--format", "{{.Size}}"
+            "images",
+            f"{registry}/{owner}/{short_image_name}:latest",
+            "--format",
+            "{{.Size}}",
         ]().rstrip()
 
         return "\n".join(
@@ -39,7 +44,7 @@ def create_header(short_image_name: str, owner: str, build_timestamp: str) -> st
                 "## Build Info",
                 "",
                 f"* Build datetime: {build_timestamp}",
-                f"* Docker image: {owner}/{short_image_name}:{commit_hash_tag}",
+                f"* Docker image: {registry}/{owner}/{short_image_name}:{commit_hash_tag}",
                 f"* Docker image size: {image_size}",
                 f"* Git commit SHA: [{commit_hash}](https://github.com/jupyter/docker-stacks/commit/{commit_hash})",
                 "* Git commit message:",
diff --git a/tagging/write_manifest.py b/tagging/write_manifest.py
diff --git a/tagging/write_tags_file.py b/tagging/write_tags_file.py
diff --git a/tests/run_tests.py b/tests/run_tests.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM jupyter/base-notebook`
	`1`	`+FROM docker.io/jupyter/base-notebook`
`2`	`2`
`3`	`3`	`# Name your environment and choose the python version`
`4`	`4`	`ARG env_name=python310`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM jupyter/base-notebook:notebook-6.5.4`
	`1`	`+FROM docker.io/jupyter/base-notebook:notebook-6.5.4`
`2`	`2`
`3`	`3`	`RUN pip install --no-cache-dir 'jupyter_contrib_nbextensions' && \`
`4`	`4`	`jupyter contrib nbextension install --user && \`