Skip to content

Commit 83063e5

Browse files
eramongodberamongodb
authored
CXX-3265 Update release instructions to use Amazon ECR instead of Artifactory (#1425)
* Use inline script metadata for extras/docker/generate.py * Migrate EVG tasks from Artifactory to Amazon ECR
1 parent 40974b5 commit 83063e5

File tree

24 files changed

+261
-158
lines changed

24 files changed

+261
-158
lines changed

‎.evergreen/config_generator/components/docker_build.py

Lines changed: 39 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from shrub.v3.evg_build_variant import BuildVariant
88
from shrub.v3.evg_command import EvgCommandType
99
from shrub.v3.evg_task import EvgTask, EvgTaskRef
10+
from shrub.v3.evg_command import KeyValueParam, ec2_assume_role, expansions_update
1011

1112

1213
TAG = 'docker-build'
@@ -24,24 +25,43 @@
2425

2526
class DockerImageBuild(Function):
2627
name = 'docker-image-build'
27-
commands = bash_exec(
28-
command_type=EvgCommandType.TEST,
29-
working_dir='mongo-cxx-driver',
30-
script='''\
31-
set -o errexit
32-
set -o pipefail
33-
docker login -u "${ARTIFACTORY_USER}" --password-stdin artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD}"
34-
set -x
35-
echo "Building Alpine Docker image"
36-
make -C extras/docker/alpine3.19 nocachebuild test
37-
echo "Building Debian Docker image"
38-
make -C extras/docker/bookworm nocachebuild test
39-
echo "Building Red Hat UBI Docker image"
40-
make -C extras/docker/redhat-ubi-9.4 nocachebuild test
41-
echo "Building Ubuntu Docker image"
42-
make -C extras/docker/noble nocachebuild test
43-
'''
44-
)
28+
commands = [
29+
# Avoid inadvertently using a pre-existing and potentially conflicting Docker config.
30+
expansions_update(updates=[KeyValueParam(key='DOCKER_CONFIG', value='${workdir}/.docker')]),
31+
ec2_assume_role(role_arn='arn:aws:iam::901841024863:role/ecr-role-evergreen-ro'),
32+
bash_exec(
33+
command_type=EvgCommandType.SETUP,
34+
include_expansions_in_env=[
35+
"AWS_ACCESS_KEY_ID",
36+
"AWS_SECRET_ACCESS_KEY",
37+
"AWS_SESSION_TOKEN",
38+
"DOCKER_CONFIG",
39+
],
40+
script='aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com',
41+
),
42+
bash_exec(
43+
command_type=EvgCommandType.TEST,
44+
working_dir='mongo-cxx-driver',
45+
env={
46+
# Use Amazon ECR as pull-through cache for DockerHub to avoid rate limits.
47+
"DEFAULT_SEARCH_REGISTRY": "901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub",
48+
},
49+
include_expansions_in_env=['DOCKER_CONFIG'],
50+
script='''\
51+
set -o errexit
52+
set -o pipefail
53+
set -x
54+
echo "Building Alpine Docker image"
55+
make -C extras/docker/alpine3.19 nocachebuild test
56+
echo "Building Debian Docker image"
57+
make -C extras/docker/bookworm nocachebuild test
58+
echo "Building Red Hat UBI Docker image"
59+
make -C extras/docker/redhat-ubi-9.4 nocachebuild test
60+
echo "Building Ubuntu Docker image"
61+
make -C extras/docker/noble nocachebuild test
62+
''',
63+
),
64+
]
4565

4666

4767
def functions():
@@ -62,7 +82,7 @@ def tasks():
6282
commands=[
6383
Setup.call(),
6484
DockerImageBuild.call(),
65-
]
85+
],
6686
)
6787
)
6888

‎.evergreen/config_generator/components/sbom.py

Lines changed: 47 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,17 @@
55
from config_generator.etc.utils import bash_exec
66

77
from shrub.v3.evg_build_variant import BuildVariant
8-
from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, ec2_assume_role, expansions_update, s3_put
8+
from shrub.v3.evg_command import (
9+
BuiltInCommand,
10+
EvgCommandType,
11+
KeyValueParam,
12+
ec2_assume_role,
13+
expansions_update,
14+
s3_put,
15+
)
916
from shrub.v3.evg_task import EvgTask, EvgTaskRef
1017

1118
from pydantic import ConfigDict
12-
from typing import Optional
1319

1420

1521
TAG = 'sbom'
@@ -23,31 +29,50 @@ class CustomCommand(BuiltInCommand):
2329
class CheckAugmentedSBOM(Function):
2430
name = 'check augmented sbom'
2531
commands = [
26-
ec2_assume_role(
27-
command_type=EvgCommandType.SETUP,
28-
role_arn='${KONDUKTO_ROLE_ARN}',
29-
),
30-
bash_exec(
31-
command_type=EvgCommandType.SETUP,
32-
include_expansions_in_env=['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN'],
33-
script='''\
34-
set -o errexit
35-
set -o pipefail
36-
kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)"
37-
printf "KONDUKTO_TOKEN: %s\\n" "$kondukto_token" >|expansions.kondukto.yml
38-
''',
39-
),
40-
expansions_update(
41-
command_type=EvgCommandType.SETUP,
42-
file='expansions.kondukto.yml',
43-
),
32+
# Authenticate with Kondukto.
33+
*[
34+
ec2_assume_role(
35+
command_type=EvgCommandType.SETUP,
36+
role_arn='${KONDUKTO_ROLE_ARN}',
37+
),
38+
bash_exec(
39+
command_type=EvgCommandType.SETUP,
40+
include_expansions_in_env=['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN'],
41+
script='''\
42+
set -o errexit
43+
set -o pipefail
44+
kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)"
45+
printf "KONDUKTO_TOKEN: %s\\n" "$kondukto_token" >|expansions.kondukto.yml
46+
''',
47+
),
48+
expansions_update(
49+
command_type=EvgCommandType.SETUP,
50+
file='expansions.kondukto.yml',
51+
),
52+
],
53+
# Authenticate with Amazon ECR.
54+
*[
55+
# Avoid inadvertently using a pre-existing and potentially conflicting Docker config.
56+
# Note: podman understands and uses DOCKER_CONFIG despite the name.
57+
expansions_update(updates=[KeyValueParam(key='DOCKER_CONFIG', value='${workdir}/.docker')]),
58+
ec2_assume_role(role_arn='arn:aws:iam::901841024863:role/ecr-role-evergreen-ro'),
59+
bash_exec(
60+
command_type=EvgCommandType.SETUP,
61+
include_expansions_in_env=[
62+
'AWS_ACCESS_KEY_ID',
63+
'AWS_SECRET_ACCESS_KEY',
64+
'AWS_SESSION_TOKEN',
65+
'DOCKER_CONFIG',
66+
],
67+
script='aws ecr get-login-password --region us-east-1 | podman login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com',
68+
),
69+
],
4470
bash_exec(
4571
command_type=EvgCommandType.TEST,
4672
working_dir='mongo-cxx-driver',
4773
include_expansions_in_env=[
48-
'ARTIFACTORY_PASSWORD',
49-
'ARTIFACTORY_USER',
5074
'branch_name',
75+
'DOCKER_CONFIG',
5176
'KONDUKTO_TOKEN',
5277
],
5378
script='.evergreen/scripts/sbom.sh',

‎.evergreen/generated_configs/functions.yml

Lines changed: 62 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -246,15 +246,33 @@ functions:
246246
type: setup
247247
params:
248248
file: expansions.kondukto.yml
249+
- command: expansions.update
250+
params:
251+
updates:
252+
- { key: DOCKER_CONFIG, value: "${workdir}/.docker" }
253+
- command: ec2.assume_role
254+
params:
255+
role_arn: arn:aws:iam::901841024863:role/ecr-role-evergreen-ro
256+
- command: subprocess.exec
257+
type: setup
258+
params:
259+
binary: bash
260+
include_expansions_in_env:
261+
- AWS_ACCESS_KEY_ID
262+
- AWS_SECRET_ACCESS_KEY
263+
- AWS_SESSION_TOKEN
264+
- DOCKER_CONFIG
265+
args:
266+
- -c
267+
- aws ecr get-login-password --region us-east-1 | podman login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com
249268
- command: subprocess.exec
250269
type: test
251270
params:
252271
binary: bash
253272
working_dir: mongo-cxx-driver
254273
include_expansions_in_env:
255-
- ARTIFACTORY_PASSWORD
256-
- ARTIFACTORY_USER
257274
- branch_name
275+
- DOCKER_CONFIG
258276
- KONDUKTO_TOKEN
259277
args:
260278
- -c
@@ -328,26 +346,48 @@ functions:
328346
- -c
329347
- .evergreen/scripts/compile.sh
330348
docker-image-build:
331-
command: subprocess.exec
332-
type: test
333-
params:
334-
binary: bash
335-
working_dir: mongo-cxx-driver
336-
args:
337-
- -c
338-
- |
339-
set -o errexit
340-
set -o pipefail
341-
docker login -u "${ARTIFACTORY_USER}" --password-stdin artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD}"
342-
set -x
343-
echo "Building Alpine Docker image"
344-
make -C extras/docker/alpine3.19 nocachebuild test
345-
echo "Building Debian Docker image"
346-
make -C extras/docker/bookworm nocachebuild test
347-
echo "Building Red Hat UBI Docker image"
348-
make -C extras/docker/redhat-ubi-9.4 nocachebuild test
349-
echo "Building Ubuntu Docker image"
350-
make -C extras/docker/noble nocachebuild test
349+
- command: expansions.update
350+
params:
351+
updates:
352+
- { key: DOCKER_CONFIG, value: "${workdir}/.docker" }
353+
- command: ec2.assume_role
354+
params:
355+
role_arn: arn:aws:iam::901841024863:role/ecr-role-evergreen-ro
356+
- command: subprocess.exec
357+
type: setup
358+
params:
359+
binary: bash
360+
include_expansions_in_env:
361+
- AWS_ACCESS_KEY_ID
362+
- AWS_SECRET_ACCESS_KEY
363+
- AWS_SESSION_TOKEN
364+
- DOCKER_CONFIG
365+
args:
366+
- -c
367+
- aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 901841024863.dkr.ecr.us-east-1.amazonaws.com
368+
- command: subprocess.exec
369+
type: test
370+
params:
371+
binary: bash
372+
working_dir: mongo-cxx-driver
373+
env:
374+
DEFAULT_SEARCH_REGISTRY: 901841024863.dkr.ecr.us-east-1.amazonaws.com/dockerhub
375+
include_expansions_in_env:
376+
- DOCKER_CONFIG
377+
args:
378+
- -c
379+
- |
380+
set -o errexit
381+
set -o pipefail
382+
set -x
383+
echo "Building Alpine Docker image"
384+
make -C extras/docker/alpine3.19 nocachebuild test
385+
echo "Building Debian Docker image"
386+
make -C extras/docker/bookworm nocachebuild test
387+
echo "Building Red Hat UBI Docker image"
388+
make -C extras/docker/redhat-ubi-9.4 nocachebuild test
389+
echo "Building Ubuntu Docker image"
390+
make -C extras/docker/noble nocachebuild test
351391
fetch-det:
352392
- command: subprocess.exec
353393
type: setup

‎.evergreen/scripts/sbom.sh

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,8 @@
33
set -o errexit
44
set -o pipefail
55

6-
: "${ARTIFACTORY_USER:?}"
7-
: "${ARTIFACTORY_PASSWORD:?}"
86
: "${branch_name:?}"
7+
: "${DOCKER_CONFIG:?}"
98
: "${KONDUKTO_TOKEN:?}"
109

1110
command -v podman >/dev/null || {
@@ -18,9 +17,7 @@ command -v jq >/dev/null || {
1817
exit 1
1918
}
2019

21-
podman login --password-stdin --username "${ARTIFACTORY_USER:?}" artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD:?}"
22-
23-
silkbomb="artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0"
20+
silkbomb="901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/silkbomb:2.0"
2421

2522
# Ensure latest version of SilkBomb is being used.
2623
podman pull "${silkbomb:?}"

‎etc/garasign_dist_file.sh

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,8 @@ if ! command -v gpg >/dev/null; then
1919
echo "gpg is required to verify distribution tarball signature" 1>&2
2020
fi
2121

22-
artifactory_creds=~/.secrets/artifactory-creds.txt
2322
garasign_creds=~/.secrets/garasign-creds.txt
2423

25-
unset ARTIFACTORY_USER ARTIFACTORY_PASSWORD
26-
# shellcheck source=/dev/null
27-
. "${artifactory_creds:?}"
28-
: "${ARTIFACTORY_USER:?"missing ARTIFACTORY_USER in ${artifactory_creds:?}"}"
29-
: "${ARTIFACTORY_PASSWORD:?"missing ARTIFACTORY_PASSWORD in ${artifactory_creds:?}"}"
30-
3124
unset GRS_CONFIG_USER1_USERNAME GRS_CONFIG_USER1_PASSWORD
3225
# shellcheck source=/dev/null
3326
. "${garasign_creds:?}"
@@ -37,10 +30,8 @@ unset GRS_CONFIG_USER1_USERNAME GRS_CONFIG_USER1_PASSWORD
3730
dist_file="${1:?}"
3831
dist_file_signed="${dist_file:?}.asc"
3932

40-
"${launcher:?}" login --password-stdin --username "${ARTIFACTORY_USER:?}" artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD:?}"
41-
4233
# Ensure latest version of Garasign is being used.
43-
"${launcher:?}" pull artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-gpg
34+
"${launcher:?}" pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/garasign-gpg
4435

4536
plugin_commands=(
4637
gpg --yes -v --armor -o "${dist_file_signed:?}" --detach-sign "${dist_file:?}"
@@ -51,7 +42,7 @@ plugin_commands=(
5142
--rm \
5243
-v "$(pwd):$(pwd)" \
5344
-w "$(pwd)" \
54-
artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-gpg
45+
901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/garasign-gpg
5546

5647
# Validate the signature file works as intended.
5748
(

‎etc/garasign_release_tag.sh

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,16 @@ if ! command -v "${launcher:?}" >/dev/null; then
1717
echo "${launcher:?} is required to create a GPG-signed release tag" 1>&2
1818
fi
1919

20-
artifactory_creds=~/.secrets/artifactory-creds.txt
2120
garasign_creds=~/.secrets/garasign-creds.txt
2221

23-
unset ARTIFACTORY_USER ARTIFACTORY_PASSWORD
24-
# shellcheck source=/dev/null
25-
. "${artifactory_creds:?}"
26-
: "${ARTIFACTORY_USER:?"missing ARTIFACTORY_USER in ${artifactory_creds:?}"}"
27-
: "${ARTIFACTORY_PASSWORD:?"missing ARTIFACTORY_PASSWORD in ${artifactory_creds:?}"}"
28-
2922
unset GRS_CONFIG_USER1_USERNAME GRS_CONFIG_USER1_PASSWORD
3023
# shellcheck source=/dev/null
3124
. "${garasign_creds:?}"
3225
: "${GRS_CONFIG_USER1_USERNAME:?"missing GRS_CONFIG_USER1_USERNAME in ${garasign_creds:?}"}"
3326
: "${GRS_CONFIG_USER1_PASSWORD:?"missing GRS_CONFIG_USER1_PASSWORD in ${garasign_creds:?}"}"
3427

35-
"${launcher:?}" login --password-stdin --username "${ARTIFACTORY_USER:?}" artifactory.corp.mongodb.com <<<"${ARTIFACTORY_PASSWORD:?}"
36-
3728
# Ensure latest version of Garasign is being used.
38-
"${launcher:?}" pull artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-git
29+
"${launcher:?}" pull 901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/garasign-git
3930

4031
# Sign using "MongoDB C++ Release Signing Key <[email protected]>" from https://pgp.mongodb.com/ (cpp-driver).
4132
git_tag_command=(
@@ -57,7 +48,7 @@ plugin_commands+=" && ${git_tag_command[*]:?}"
5748
--rm \
5849
-v "$(pwd):$(pwd)" \
5950
-w "$(pwd)" \
60-
artifactory.corp.mongodb.com/release-tools-container-registry-local/garasign-git
51+
901841024863.dkr.ecr.us-east-1.amazonaws.com/release-infrastructure/garasign-git
6152

6253
# Validate the release tag is signed as intended.
6354
(

0 commit comments

Comments
 (0)