CDRIVER-5971 Use Amazon ECR to obtain OCI images in EVG

.evergreen/config_generator/components/earthly.py

@@ -5,9 +5,18 @@
 from typing import Iterable, Literal, Mapping, NamedTuple, TypeVar
 
 from shrub.v3.evg_build_variant import BuildVariant
-from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, subprocess_exec
+from shrub.v3.evg_command import (
+    BuiltInCommand,
+    EvgCommandType,
+    KeyValueParam,
+    ec2_assume_role,
+    expansions_update,
+    subprocess_exec,
+)
 from shrub.v3.evg_task import EvgTask, EvgTaskRef
 
+from config_generator.etc.function import Function
+
 from ..etc.utils import all_possible
 
 T = TypeVar("T")
@@ -38,7 +47,7 @@
 "Valid options for the SASL configuration parameter"
 TLSOption = Literal["OpenSSL", "off"]
 "Options for the TLS backend configuration parameter (AKA 'ENABLE_SSL')"
-CxxVersion = Literal["none"] # TODO: Once CXX-3103 is released, add latest C++ release tag.
+CxxVersion = Literal["none"]  # TODO: Once CXX-3103 is released, add latest C++ release tag.
 "C++ driver refs that are under CI test"
 
 # A separator character, since we cannot use whitespace
@@ -136,6 +145,34 @@ def suffix(self) -> str:
         return _SEPARATOR.join(f"{k}={v}" for k, v in self._asdict().items())
 
 
+# Use DevProd-provided Amazon ECR instance to obtain earthly-buildkitd in advance.
+class DockerLoginAmazonECR(Function):
+    name = 'docker-login-amazon-ecr'
+    commands = [
+        # Avoid inadvertently using a pre-existing and potentially conflicting Docker config.
+        expansions_update(updates=[KeyValueParam(key='DOCKER_CONFIG', value='${workdir}/.docker')]),
+        ec2_assume_role(role_arn="arn:aws:iam::901841024863:role/ecr-role-evergreen-ro"),
+        subprocess_exec(
+            binary="bash",
+            command_type=EvgCommandType.SETUP,
+            include_expansions_in_env=[
+                "AWS_ACCESS_KEY_ID",
+                "AWS_SECRET_ACCESS_KEY",
+                "AWS_SESSION_TOKEN",
+                "DOCKER_CONFIG",
+            ],
+            args=[
+                "-c",
+                'aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com',
+            ],
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
 def task_filter(env: EarthlyVariant, conf: Configuration) -> bool:
     """
     Control which tasks are actually defined by matching on the platform and
@@ -175,6 +212,7 @@ def earthly_exec(
             *(f"--{arg}={val}" for arg, val in (args or {}).items()),
         ],
         command_type=EvgCommandType(kind),
+        include_expansions_in_env=["DOCKER_CONFIG"],
         env=env if env else None,
         working_dir="mongoc",
     )
@@ -209,10 +247,12 @@ def earthly_task(
     return EvgTask(
         name=name,
         commands=[
+            DockerLoginAmazonECR.call(),
             # Ensure subsequent Docker commands are authenticated.
             subprocess_exec(
                 binary="bash",
                 command_type=EvgCommandType.SETUP,
+                include_expansions_in_env=["DOCKER_CONFIG"],
                 args=[
                     "-c",
                     r'docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}"',
@@ -249,6 +289,10 @@ def earthly_task(
 ]
 
 
+def functions():
+    return DockerLoginAmazonECR.defn()
+
+
 def tasks() -> Iterable[EvgTask]:
     for conf in all_possible(Configuration):
         # test-example is a target in all configurations

.evergreen/generated_configs/functions.yml

@@ -123,7 +123,7 @@ functions:
           DRYRUN: "1"
         args:
           - -c
-          - uv run --frozen --only-group format tools/format.py --mode=check
+          - uv run --frozen --only-group=format tools/format.py --mode=check
   cse-sasl-cyrus-darwinssl-compile:
     - command: expansions.update
       params:
@@ -175,6 +175,26 @@ functions:
         args:
           - -c
           - EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON ${EXTRA_CONFIGURE_FLAGS}" .evergreen/scripts/compile.sh
+  docker-login-amazon-ecr:
+    - command: expansions.update
+      params:
+        updates:
+          - { key: DOCKER_CONFIG, value: "${workdir}/.docker" }
+    - command: ec2.assume_role
+      params:
+        role_arn: arn:aws:iam::901841024863:role/ecr-role-evergreen-ro
+    - command: subprocess.exec
+      type: setup
+      params:
+        binary: bash
+        include_expansions_in_env:
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - AWS_SESSION_TOKEN
+          - DOCKER_CONFIG
+        args:
+          - -c
+          - aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com
   fetch-build:
     - command: subprocess.exec
       type: setup

.evergreen/generated_configs/tasks.yml

@@ -1133,10 +1133,13 @@ tasks:
       - ubuntu2404-large
     tags: [earthly, pr-merge-gate, alpine3.16-clang, alpine3.16-gcc, alpine3.17-clang, alpine3.17-gcc, alpine3.18-clang, alpine3.18-gcc, alpine3.19-clang, alpine3.19-gcc, archlinux-clang, archlinux-gcc, centos7-clang, centos7-gcc, u16-clang, u16-gcc, u18-clang, u18-gcc, u20-clang, u20-gcc, u22-clang, u22-gcc]
     commands:
+      - func: docker-login-amazon-ecr
       - command: subprocess.exec
         type: setup
         params:
           binary: bash
+          include_expansions_in_env:
+            - DOCKER_CONFIG
           args:
             - -c
             - docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}"
@@ -1145,6 +1148,8 @@ tasks:
         params:
           binary: ./tools/earthly.sh
           working_dir: mongoc
+          include_expansions_in_env:
+            - DOCKER_CONFIG
           args:
             - +env-warmup
             - --sasl=Cyrus
@@ -1157,6 +1162,8 @@ tasks:
         params:
           binary: ./tools/earthly.sh
           working_dir: mongoc
+          include_expansions_in_env:
+            - DOCKER_CONFIG
           args:
             - +run
             - --targets=test-example
@@ -1174,10 +1181,13 @@ tasks:
       - ubuntu2404-large
     tags: [earthly, pr-merge-gate, alpine3.16-clang, alpine3.16-gcc, alpine3.17-clang, alpine3.17-gcc, alpine3.18-clang, alpine3.18-gcc, alpine3.19-clang, alpine3.19-gcc, archlinux-clang, archlinux-gcc, centos7-clang, centos7-gcc, u16-clang, u16-gcc, u18-clang, u18-gcc, u20-clang, u20-gcc, u22-clang, u22-gcc]
     commands:
+      - func: docker-login-amazon-ecr
       - command: subprocess.exec
         type: setup
         params:
           binary: bash
+          include_expansions_in_env:
+            - DOCKER_CONFIG
           args:
             - -c
             - docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}"
@@ -1186,6 +1196,8 @@ tasks:
         params:
           binary: ./tools/earthly.sh
           working_dir: mongoc
+          include_expansions_in_env:
+            - DOCKER_CONFIG
           args:
             - +env-warmup
             - --sasl=Cyrus
@@ -1198,6 +1210,8 @@ tasks:
         params:
           binary: ./tools/earthly.sh
           working_dir: mongoc
+          include_expansions_in_env:
+            - DOCKER_CONFIG
           args:
             - +run
             - --targets=test-example
@@ -1215,10 +1229,13 @@ tasks:
       - ubuntu2404-large
     tags: [earthly, pr-merge-gate, alpine3.16-clang, alpine3.16-gcc, alpine3.17-clang, alpine3.17-gcc, alpine3.18-clang, alpine3.18-gcc, alpine3.19-clang, alpine3.19-gcc, archlinux-clang, archlinux-gcc, centos7-clang, centos7-gcc, u16-clang, u16-gcc, u18-clang, u18-gcc, u20-clang, u20-gcc, u22-clang, u22-gcc]
     commands:
+      - func: docker-login-amazon-ecr
       - command: subprocess.exec
         type: setup
         params:
           binary: bash
+          include_expansions_in_env:
+            - DOCKER_CONFIG
           args:
             - -c
             - docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}"
@@ -1227,6 +1244,8 @@ tasks:
         params:
           binary: ./tools/earthly.sh
           working_dir: mongoc
+          include_expansions_in_env:
+            - DOCKER_CONFIG
           args:
             - +env-warmup
             - --sasl=off
@@ -1239,6 +1258,8 @@ tasks:
         params:
           binary: ./tools/earthly.sh
           working_dir: mongoc
+          include_expansions_in_env:
+            - DOCKER_CONFIG
           args:
             - +run
             - --targets=test-example
@@ -1256,10 +1277,13 @@ tasks:
       - ubuntu2404-large
     tags: [earthly, pr-merge-gate, alpine3.16-clang, alpine3.16-gcc, alpine3.17-clang, alpine3.17-gcc, alpine3.18-clang, alpine3.18-gcc, alpine3.19-clang, alpine3.19-gcc, archlinux-clang, archlinux-gcc, centos7-clang, centos7-gcc, u16-clang, u16-gcc, u18-clang, u18-gcc, u20-clang, u20-gcc, u22-clang, u22-gcc]
     commands:
+      - func: docker-login-amazon-ecr
       - command: subprocess.exec
         type: setup
         params:
           binary: bash
+          include_expansions_in_env:
+            - DOCKER_CONFIG
           args:
             - -c
             - docker login -u "${artifactory_username}" --password-stdin artifactory.corp.mongodb.com <<<"${artifactory_password}"
@@ -1268,6 +1292,8 @@ tasks:
         params:
           binary: ./tools/earthly.sh
           working_dir: mongoc
+          include_expansions_in_env:
+            - DOCKER_CONFIG
           args:
             - +env-warmup
             - --sasl=off
@@ -1280,6 +1306,8 @@ tasks:
         params:
           binary: ./tools/earthly.sh
           working_dir: mongoc
+          include_expansions_in_env:
+            - DOCKER_CONFIG
           args:
             - +run
             - --targets=test-example

Earthfile

@@ -1,4 +1,4 @@
-VERSION --arg-scope-and-set --pass-args 0.7
+VERSION --arg-scope-and-set --pass-args --use-function-keyword 0.7
 LOCALLY
 
 IMPORT ./tools/ AS tools
@@ -114,7 +114,7 @@ test-cxx-driver:
 
 # PREP_CMAKE "warms up" the CMake installation cache for the current environment
 PREP_CMAKE:
-    COMMAND
+    FUNCTION
     LET scratch=/opt/mongoc-cmake
     # Copy the minimal amount that we need, as to avoid cache invalidation
     COPY tools/use.sh tools/platform.sh tools/paths.sh tools/base.sh tools/download.sh \
@@ -148,7 +148,7 @@ multibuild:
 # release-archive :
 #   Create a release archive of the source tree. (Refer to dev docs)
 release-archive:
-    FROM alpine:3.20
+    FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:3.20
     RUN apk add git bash
     ARG --required prefix
     ARG --required ref
@@ -193,7 +193,7 @@ release-archive:
 
 # Obtain the signing public key. Exported as an artifact /c-driver.pub
 signing-pubkey:
-    FROM alpine:3.20
+    FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:3.20
     RUN apk add curl
     RUN curl --location --silent --fail "https://pgp.mongodb.com/c-driver.pub" -o /c-driver.pub
     SAVE ARTIFACT /c-driver.pub
@@ -203,7 +203,7 @@ signing-pubkey:
 #   to be used to access them. (Refer to dev docs)
 sign-file:
     # Pull from Garasign:
-    FROM artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-gpg
+    FROM artifactory.corp.mongodb.com/release-infrastructure/garasign-gpg
     # Copy the file to be signed
     ARG --required file
     COPY $file /s/file
@@ -223,7 +223,7 @@ sign-file:
 #   Generate a signed release artifact. Refer to the "Earthly" page of our dev docs for more information.
 #   (Refer to dev docs)
 signed-release:
-    FROM alpine:3.20
+    FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:3.20
     RUN apk add git
     # The version of the release. This affects the filepaths of the output and is the default for --ref
     ARG --required version
@@ -312,7 +312,7 @@ sbom-validate:
             --exclude jira
 
 snyk:
-    FROM --platform=linux/amd64 ubuntu:24.04
+    FROM --platform=linux/amd64 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/ubuntu:24.04
     RUN apt-get update && apt-get -y install curl
     RUN curl --location https://github.com/snyk/cli/releases/download/v1.1291.1/snyk-linux -o /usr/local/bin/snyk
     RUN chmod a+x /usr/local/bin/snyk
@@ -384,7 +384,7 @@ test-vcpkg-manifest-mode:
         make test-manifest-mode
 
 vcpkg-base:
-    FROM alpine:3.18
+    FROM 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:3.18
     RUN apk add cmake curl gcc g++ musl-dev ninja-is-really-ninja zip unzip tar \
                 build-base git pkgconf perl bash linux-headers
     ENV VCPKG_ROOT=/opt/vcpkg-git
@@ -443,7 +443,7 @@ env.alpine3.19:
     DO --pass-args +ALPINE_ENV --version=3.19
 
 env.archlinux:
-    FROM --pass-args tools+init-env --from archlinux
+    FROM --pass-args tools+init-env --from 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/archlinux
     RUN pacman-key --init
     ARG --required purpose
 
@@ -462,9 +462,9 @@ env.centos7:
     DO --pass-args +CENTOS_ENV --version=7
 
 ALPINE_ENV:
-    COMMAND
+    FUNCTION
     ARG --required version
-    FROM --pass-args tools+init-env --from alpine:$version
+    FROM --pass-args tools+init-env --from 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/alpine:$version
     # XXX: On Alpine, we just use the system's CMake. At time of writing, it is
     # very up-to-date and much faster than building our own from source (since
     # Kitware does not (yet) provide libmuslc builds of CMake)
@@ -484,9 +484,9 @@ ALPINE_ENV:
     DO --pass-args tools+ADD_C_COMPILER --clang_pkg="gcc clang compiler-rt"
 
 UBUNTU_ENV:
-    COMMAND
+    FUNCTION
     ARG --required version
-    FROM --pass-args tools+init-env --from ubuntu:$version
+    FROM --pass-args tools+init-env --from 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/ubuntu:$version
     RUN __install curl build-essential
     ARG --required purpose
 
@@ -502,13 +502,13 @@ UBUNTU_ENV:
     DO +PREP_CMAKE
 
 CENTOS_ENV:
-    COMMAND
+    FUNCTION
     ARG --required version
-    FROM --pass-args tools+init-env --from centos:$version
+    FROM --pass-args tools+init-env --from 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub/library/centos:$version
     # Update repositories to use vault.centos.org
     RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
         sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
-    RUN yum -y install epel-release && yum -y update
+    RUN yum -y --enablerepo=extras install epel-release && yum -y update
     RUN yum -y install curl gcc gcc-c++ make
     ARG --required purpose