From c3295890d645f360d6a7c6d770f099ef50c8ada4 Mon Sep 17 00:00:00 2001 From: Emma Doyle Date: Fri, 26 Apr 2024 16:21:03 -0400 Subject: [PATCH 1/8] PLT-37: Create an AWS ECR docker push job --- src/@orb.yml | 3 +- src/commands/docker/login.yml | 2 +- src/jobs/docker/ecr.yml | 107 ++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 src/jobs/docker/ecr.yml diff --git a/src/@orb.yml b/src/@orb.yml index e01f2b4..ddd0b38 100644 --- a/src/@orb.yml +++ b/src/@orb.yml @@ -10,4 +10,5 @@ display: orbs: helm: circleci/helm@2.0.1 - circleci-cli: circleci/circleci-cli@0.1.9 \ No newline at end of file + circleci-cli: circleci/circleci-cli@0.1.9 + aws-cli: circleci/aws-cli@4.1.3 \ No newline at end of file diff --git a/src/commands/docker/login.yml b/src/commands/docker/login.yml index 81e4334..3484b32 100644 --- a/src/commands/docker/login.yml +++ b/src/commands/docker/login.yml @@ -1,5 +1,5 @@ description: | - Login to Docker registry + Login to a Nexus-like Docker registry parameters: username: description: Docker username. diff --git a/src/jobs/docker/ecr.yml b/src/jobs/docker/ecr.yml new file mode 100644 index 0000000..633a5b2 --- /dev/null +++ b/src/jobs/docker/ecr.yml @@ -0,0 +1,107 @@ +description: |+ + Build an push a Docker image to AWS ECR. + + Requires a context with $AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, $AWS_DEFAULT_REGION, $AWS_ACCOUNT_ID set. +executor: + name: machine + caching: << parameters.docker-layer-caching >> +resource_class: << parameters.resource-class >> +parameters: + resource-class: + type: enum + enum: + - medium + - large + - xlarge + - 2xlarge + default: medium + description: Resource class to run as. + image-name: + description: Name of the image. + default: '' + type: string + tag: + description: Name of the tag for the image. + default: latest + type: string + commit-tag: + description: Whether or not to push an additional tag to the registry with the commit hash as the tag. + default: false + type: boolean + docker-layer-caching: + description: Enable DLC on the machine executor. Costs 200 credits / run, however. + default: false + type: boolean + args: + description: Additional args string to add to the build command. (E.g., '--build-arg=HELLO=WORLD'.) + default: '' + type: string + path: + description: Path to a particular Dockerfile or containing directory with a Dockerfile present. + default: Dockerfile + type: string + aws-access-key-id: + description: AWS Access Key ID. + type: string + default: $AWS_ACCESS_KEY_ID + aws-secret-access-key: + description: AWS Secret Access Key ID. + type: string + default: $AWS_SECRET_ACCESS_KEY + aws-region: + description: Region of the ECR registry. + type: string + default: $AWS_DEFAULT_REGION + aws-account-id: + description: AWS Account ID. + type: string + default: $AWS_ACCOUNT_ID + aws-ecr-name: + description: AWS ECR (registry) name. + type: string + aws-cli-version: + description: Version of the AWS CLI to install. + type: string + default: latest + buildkit: + description: Enable buildkit (https://docs.docker.com/build/buildkit/#getting-started). + type: boolean + default: true +steps: + - checkout + - aws-cli/install: + version: << parameters.aws-cli-version >> + - run: + name: Docker login (ECR via 'aws ecr') + command: |+ + aws ecr get-login-password --region << parameters.aws-region >> | docker login --username AWS --password-stdin << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com + - run: + name: Build and push tag + command: |+ + if [[ "<< parameters.tag >>" =~ v[0-9]+.[0-9]+.[0-9]+ ]]; then + export _DOCKER_TAG="$(printf "%s" "<< parameters.tag >>" | grep -oP "(?<=v).*" | awk NF)" + else + export _DOCKER_TAG="<< parameters.tag >>" + fi + + if [ "<< parameters.buildkit >>" = "true" ]; then + export DOCKER_BUILDKIT=1 + fi + + if [ "$(echo "<< parameters.path >>" | grep -oP "Dockerfile")" != "Dockerfile" ]; then + # Path does not contain the dockerfile, so we default to "Dockerfile". + docker build . -f ./<< parameters.path >>/Dockerfile -t << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.args >> + docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" + else + # Path does specify the Dockerfile explictly, don't default to "Dockerfile". E.g., "docker/Dockerfile.develop". + docker build . -f ./<< parameters.path >> -t << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.args >> + docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" + fi + - when: + condition: << parameters.commit-tag >> + steps: + - run: + name: Push commit tag + command: |+ + docker tag << parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 + docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 \ No newline at end of file From a7c7025cadb05fd4fa2c7e82da4169b0d144cf5a Mon Sep 17 00:00:00 2001 From: Emma Doyle Date: Fri, 26 Apr 2024 16:35:30 -0400 Subject: [PATCH 2/8] PLT-37: Use account name as well --- src/jobs/docker/ecr.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/jobs/docker/ecr.yml b/src/jobs/docker/ecr.yml index 633a5b2..439df1f 100644 --- a/src/jobs/docker/ecr.yml +++ b/src/jobs/docker/ecr.yml @@ -59,6 +59,7 @@ parameters: aws-ecr-name: description: AWS ECR (registry) name. type: string + default: develop aws-cli-version: description: Version of the AWS CLI to install. type: string @@ -90,12 +91,12 @@ steps: if [ "$(echo "<< parameters.path >>" | grep -oP "Dockerfile")" != "Dockerfile" ]; then # Path does not contain the dockerfile, so we default to "Dockerfile". - docker build . -f ./<< parameters.path >>/Dockerfile -t << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.args >> - docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" + docker build . -f ./<< parameters.path >>/Dockerfile -t << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.aws-ecr-name >>/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.args >> + docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.aws-ecr-name >>/<< parameters.image-name >>:"$_DOCKER_TAG" else # Path does specify the Dockerfile explictly, don't default to "Dockerfile". E.g., "docker/Dockerfile.develop". - docker build . -f ./<< parameters.path >> -t << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.args >> - docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" + docker build . -f ./<< parameters.path >> -t << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.aws-ecr-name >>/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.args >> + docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.aws-ecr-name >>/<< parameters.image-name >>:"$_DOCKER_TAG" fi - when: condition: << parameters.commit-tag >> @@ -103,5 +104,5 @@ steps: - run: name: Push commit tag command: |+ - docker tag << parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 - docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 \ No newline at end of file + docker tag << parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.aws-ecr-name >>/<< parameters.image-name >>:$CIRCLE_SHA1 + docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.aws-ecr-name >>/<< parameters.image-name >>:$CIRCLE_SHA1 \ No newline at end of file From 76891b1662e505d08f4dc431ef5edbb4002c868f Mon Sep 17 00:00:00 2001 From: Emma Doyle Date: Fri, 26 Apr 2024 18:19:19 -0400 Subject: [PATCH 3/8] PLT-37: Fix target ecr registry creation --- src/jobs/docker/ecr.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/jobs/docker/ecr.yml b/src/jobs/docker/ecr.yml index 439df1f..da30a69 100644 --- a/src/jobs/docker/ecr.yml +++ b/src/jobs/docker/ecr.yml @@ -17,7 +17,7 @@ parameters: default: medium description: Resource class to run as. image-name: - description: Name of the image. + description: Name of the image, and also the ECR repository. default: '' type: string tag: @@ -56,10 +56,6 @@ parameters: description: AWS Account ID. type: string default: $AWS_ACCOUNT_ID - aws-ecr-name: - description: AWS ECR (registry) name. - type: string - default: develop aws-cli-version: description: Version of the AWS CLI to install. type: string @@ -76,6 +72,16 @@ steps: name: Docker login (ECR via 'aws ecr') command: |+ aws ecr get-login-password --region << parameters.aws-region >> | docker login --username AWS --password-stdin << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com + - run: + name: Check ECR repository target + command: |+ + # If it does not exist, just create it. + if [ -z "$(aws ecr describe-repositories | jq -rMC ".repositories[] | select(.repositoryName == \"<< parameters.image-name >>\") | .repositoryName")" ]; then + printf "WARNING: Repository \"<< parameters.image-name >>\" does not exist. Creating.\\n" >&2 + aws ecr create-repository --repository-name "<< parameters.image-name >>" + else + printf "INFO: Repository \"<< parameters.image-name >>\" already exists.\\n" + fi - run: name: Build and push tag command: |+ From 6db0ff051999955508bd1cdb04aaecb3091976d8 Mon Sep 17 00:00:00 2001 From: Emma Doyle Date: Fri, 26 Apr 2024 18:24:18 -0400 Subject: [PATCH 4/8] PLT-37: Update job to default to repository name of image-name --- src/jobs/docker/ecr.yml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/jobs/docker/ecr.yml b/src/jobs/docker/ecr.yml index da30a69..3999027 100644 --- a/src/jobs/docker/ecr.yml +++ b/src/jobs/docker/ecr.yml @@ -28,6 +28,10 @@ parameters: description: Whether or not to push an additional tag to the registry with the commit hash as the tag. default: false type: boolean + branch-tag: + description: Whether or not to push an additional tag to the registry with the branch name as the tag. + default: false + type: boolean docker-layer-caching: description: Enable DLC on the machine executor. Costs 200 credits / run, however. default: false @@ -97,12 +101,12 @@ steps: if [ "$(echo "<< parameters.path >>" | grep -oP "Dockerfile")" != "Dockerfile" ]; then # Path does not contain the dockerfile, so we default to "Dockerfile". - docker build . -f ./<< parameters.path >>/Dockerfile -t << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.aws-ecr-name >>/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.args >> - docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.aws-ecr-name >>/<< parameters.image-name >>:"$_DOCKER_TAG" + docker build . -f ./<< parameters.path >>/Dockerfile -t << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.args >> + docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" else # Path does specify the Dockerfile explictly, don't default to "Dockerfile". E.g., "docker/Dockerfile.develop". - docker build . -f ./<< parameters.path >> -t << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.aws-ecr-name >>/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.args >> - docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.aws-ecr-name >>/<< parameters.image-name >>:"$_DOCKER_TAG" + docker build . -f ./<< parameters.path >> -t << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.args >> + docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" fi - when: condition: << parameters.commit-tag >> @@ -110,5 +114,13 @@ steps: - run: name: Push commit tag command: |+ - docker tag << parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.aws-ecr-name >>/<< parameters.image-name >>:$CIRCLE_SHA1 - docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.aws-ecr-name >>/<< parameters.image-name >>:$CIRCLE_SHA1 \ No newline at end of file + docker tag << parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 + docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 + - when: + condition: << parameters.branch-tag >> + steps: + - run: + name: Push branch tag + command: |+ + docker tag << parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_BRANCH + docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_BRANCH \ No newline at end of file From a7b0632d9fb3973693126e749ba43396b331cbba Mon Sep 17 00:00:00 2001 From: Emma Doyle Date: Fri, 26 Apr 2024 19:27:59 -0400 Subject: [PATCH 5/8] PLT-37: Fix follow-up tagging --- src/jobs/docker/ecr.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jobs/docker/ecr.yml b/src/jobs/docker/ecr.yml index 3999027..b93f425 100644 --- a/src/jobs/docker/ecr.yml +++ b/src/jobs/docker/ecr.yml @@ -114,7 +114,7 @@ steps: - run: name: Push commit tag command: |+ - docker tag << parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 + docker tag << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 - when: condition: << parameters.branch-tag >> @@ -122,5 +122,5 @@ steps: - run: name: Push branch tag command: |+ - docker tag << parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_BRANCH + docker tag << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_BRANCH docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_BRANCH \ No newline at end of file From 8914df2adf0e2ca7f6d498170d7050067e4db844 Mon Sep 17 00:00:00 2001 From: Emma Doyle Date: Fri, 26 Apr 2024 19:28:52 -0400 Subject: [PATCH 6/8] PLT-37: Minor fix for another docker tag follow-up job --- src/jobs/docker/hub.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobs/docker/hub.yml b/src/jobs/docker/hub.yml index 2f500a5..19759f8 100644 --- a/src/jobs/docker/hub.yml +++ b/src/jobs/docker/hub.yml @@ -90,5 +90,5 @@ steps: - run: name: Push commit tag command: |+ - docker tag << parameters.image-name >>:"$_DOCKER_TAG" << parameters.organization >>/<< parameters.image-name >>:$CIRCLE_SHA1 + docker tag << parameters.organization >>/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.organization >>/<< parameters.image-name >>:$CIRCLE_SHA1 docker push << parameters.organization >>/<< parameters.image-name >>:$CIRCLE_SHA1 \ No newline at end of file From c4a6fde0c2c26882c3e03c9d7182589b1eeebfcd Mon Sep 17 00:00:00 2001 From: Emma Doyle Date: Fri, 26 Apr 2024 19:36:40 -0400 Subject: [PATCH 7/8] PLT-37: Bump version of machine executor --- src/executors/machine.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/executors/machine.yml b/src/executors/machine.yml index 3a78f20..4d420e7 100644 --- a/src/executors/machine.yml +++ b/src/executors/machine.yml @@ -1,7 +1,7 @@ description: Docker build / machine executor. parameters: tag: - default: 2204:2022.07.1 + default: 2204:2024.01.2 description: Ubuntu version string to use. type: string caching: From 625a11bf81adbdfa210f5be4b5db83f2d3700b1b Mon Sep 17 00:00:00 2001 From: Emma Doyle Date: Fri, 26 Apr 2024 19:44:15 -0400 Subject: [PATCH 8/8] PLT-37: Fix logic around tags --- src/jobs/docker/ecr.yml | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/jobs/docker/ecr.yml b/src/jobs/docker/ecr.yml index b93f425..ea8487d 100644 --- a/src/jobs/docker/ecr.yml +++ b/src/jobs/docker/ecr.yml @@ -108,19 +108,15 @@ steps: docker build . -f ./<< parameters.path >> -t << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.args >> docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" fi - - when: - condition: << parameters.commit-tag >> - steps: - - run: - name: Push commit tag - command: |+ - docker tag << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 - docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 - - when: - condition: << parameters.branch-tag >> - steps: - - run: - name: Push branch tag - command: |+ - docker tag << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_BRANCH - docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_BRANCH \ No newline at end of file + + if [ "<< parameters.commit-tag >>" = "true" ]; then + printf "INFO: Pushing Docker image tag based on git commit SHA at user's request.\\n" + docker tag << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 + docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_SHA1 + fi + + if [ "<< parameters.branch-tag >>" = "true" ]; then + printf "INFO: Pushing Docker image tag based on git branch name at user's request.\\n" + docker tag << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:"$_DOCKER_TAG" << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_BRANCH + docker push << parameters.aws-account-id >>.dkr.ecr.<< parameters.aws-region >>.amazonaws.com/<< parameters.image-name >>:$CIRCLE_BRANCH + fi \ No newline at end of file