Skip to content

Commit 5df84ac

Browse files
bistlinebistline
authored
Merge pull request #28 from broadinstitute/jb-gha-docker-tasks
Migrating Jenkins jobs to GitHub Actions (SCP-5476)
2 parents 7ea0de0 + 652fd26 commit 5df84ac

File tree

11 files changed

+313
-1
lines changed

11 files changed

+313
-1
lines changed

‎.github/actions/configure-gcloud-container/action.yml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: 'Pull gcloud container and activate service account'
2+
description: 'Pulls gcloud Docker image and activates the requested service account to use in calling gcloud'
3+
inputs:
4+
vault_secret_path:
5+
description: 'Path to service account in vault'
6+
required: true
7+
gcloud_docker_image:
8+
description: 'gcloud Docker image name/vesion tag'
9+
required: false
10+
default: 'gcr.io/google.com/cloudsdktool/google-cloud-cli:latest'
11+
image_name:
12+
description: 'name of image to create and access credentials from'
13+
required: false
14+
default: 'gcloud-config'
15+
vault_addr:
16+
description: 'vault API endpoint'
17+
required: true
18+
vault_role_id:
19+
description: 'vault role for accessing secrets'
20+
required: true
21+
vault_secret_id:
22+
description: 'credential to authenticate into vault'
23+
required: true
24+
25+
runs:
26+
using: 'composite'
27+
steps:
28+
- name: Install vault
29+
uses: ./.github/actions/install-vault-and-utils
30+
- name: Download gcloud Docker image
31+
shell: bash
32+
run: |
33+
docker pull ${{ inputs.gcloud_docker_image }}
34+
- name: Extract main service account json
35+
uses: ./.github/actions/extract-vault-secret-to-file
36+
with:
37+
vault_secret_path: ${{ inputs.vault_secret_path }}
38+
vault_addr: ${{ inputs.VAULT_ADDR }}
39+
vault_role_id: ${{ inputs.VAULT_ROLE_ID }}
40+
vault_secret_id: ${{ inputs.VAULT_SECRET_ID }}
41+
output_filename: 'scp_service_account.json'
42+
output_format: 'json'
43+
- name: Authenticate to gcloud and docker
44+
env:
45+
SERVICE_ACCOUNT_FILE: 'scp_service_account.json'
46+
shell: bash
47+
run: |
48+
# manually construct a Dockerfile using supplied image, and pre-authenticate the service account via gcloud
49+
# will also run ssh-keygen so that this doesn't execute every time we make a gcloud compute ssh call
50+
SERVICE_ACCOUNT_PATH="/tmp/$SERVICE_ACCOUNT_FILE"
51+
touch Dockerfile-gcloud
52+
echo "FROM ${{ inputs.gcloud_docker_image }}" >> Dockerfile-gcloud
53+
echo >> Dockerfile-gcloud
54+
echo "COPY $SERVICE_ACCOUNT_FILE $SERVICE_ACCOUNT_PATH" >> Dockerfile-gcloud
55+
echo "RUN gcloud auth activate-service-account --key-file=$SERVICE_ACCOUNT_PATH" >> Dockerfile-gcloud
56+
echo "RUN gcloud auth configure-docker" >> Dockerfile-gcloud
57+
echo "RUN ssh-keygen -t rsa -f /root/.ssh/google_compute_engine -C root -b 2048 -q -N \"\"" >> Dockerfile-gcloud
58+
# install NumPy for faster tunnelling and to suppress warnings
59+
echo "RUN \$(gcloud info --format=\"value(basic.python_location)\") -m pip install numpy" >> Dockerfile-gcloud
60+
# silence various ssh warnings
61+
echo "RUN echo \"LogLevel ERROR\" > /root/.ssh/config" >> Dockerfile-gcloud
62+
echo "ENV LC_ALL=C" >> Dockerfile-gcloud
63+
docker build -t ${{ inputs.image_name }} -f Dockerfile-gcloud .

‎.github/actions/extract-vault-secret-to-file/action.yml

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
name: 'Extract vault secret to file'
2+
description: 'read a secret from a path in vault and export as a file'
3+
inputs:
4+
vault_secret_path:
5+
description: 'Path to secret in vault'
6+
required: true
7+
vault_addr:
8+
description: 'vault API endpoint'
9+
required: true
10+
vault_role_id:
11+
description: 'vault role for accessing secrets'
12+
required: true
13+
vault_secret_id:
14+
description: 'credential to authenticate into vault'
15+
required: true
16+
output_filename:
17+
description: 'name of exported secrets file'
18+
required: true
19+
output_format:
20+
description: 'form to write secrets to: env or json'
21+
required: true
22+
23+
runs:
24+
using: 'composite'
25+
steps:
26+
- name: Extract vault secret to file
27+
shell: bash
28+
run: |
29+
export VAULT_ADDR=${{ inputs.vault_addr }}
30+
export VAULT_TOKEN=$( vault write -field=token auth/approle/login \
31+
role_id=${{ inputs.vault_role_id}} secret_id=${{ inputs.vault_secret_id}} )
32+
VALUES=$(vault read -format json ${{ inputs.vault_secret_path }})
33+
if [[ "${{ inputs.output_format }}" = 'env' ]]; then
34+
echo "### env secrets from ${{ inputs.vault_secret_path }} ###" >| ${{ inputs.output_filename }}
35+
for key in $(echo $VALUES | jq .data | jq --raw-output 'keys[]')
36+
do
37+
echo "setting value for: $key"
38+
curr_val=$(echo $VALUES | jq .data | jq --raw-output .$key)
39+
echo "export $key='$curr_val'" >> ${{ inputs.output_filename }}
40+
done
41+
elif [[ "${{ inputs.output_format }}" = 'json' ]]; then
42+
JSON_CONTENTS=$(echo $VALUES | jq --raw-output .data)
43+
echo $JSON_CONTENTS >| ${{ inputs.output_filename }}
44+
fi

‎.github/actions/install-vault-and-utils/action.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
name: 'Install vault and other utilities'
2+
description: 'Installs vault, curl, unzip and jq to use in other workflows'
3+
runs:
4+
using: 'composite'
5+
steps:
6+
- name: Install vault and utils
7+
shell: bash
8+
run: |
9+
sudo apt-get update && sudo apt-get -y install curl unzip jq
10+
sudo curl -O https://releases.hashicorp.com/vault/1.9.0/vault_1.9.0_linux_amd64.zip
11+
sudo unzip vault_1.9.0_linux_amd64.zip
12+
sudo mv vault /usr/local/bin

‎.github/actions/set-exit-code/action.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
name: 'Set exit code'
2+
description: 'Sets exit code of workflow using status from previous step, allowing for unstable 99 as good'
3+
inputs:
4+
exit-code:
5+
description: 'Exit code from previous step'
6+
required: true
7+
runs:
8+
using: 'composite'
9+
steps:
10+
- name: Set exit code
11+
shell: bash
12+
run: |
13+
if [[ "${{ inputs.exit-code }}" -eq "99" ]]; then
14+
echo "### WARNING: THERE ARE UPDATED DEPENDENCIES, BUT THIS IMAGE WILL STILL FUNCTION ###"
15+
echo "### MARKING AS SUCCESSFUL ###"
16+
exit 0
17+
else
18+
exit ${{ inputs.exit-code }}
19+
fi

‎.github/workflows/publish-base-docker-image.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Publish rails-baseimage Docker image
2+
on:
3+
push:
4+
branches:
5+
- master
6+
workflow_dispatch:
7+
env:
8+
VAULT_SECRET_PATH: 'secret/kdux/scp/staging/scp_service_account.json'
9+
IMAGE_NAME: 'gcloud-config'
10+
11+
jobs:
12+
Build-And-Publish-Docker-Image:
13+
runs-on: ubuntu-20.04
14+
steps:
15+
- name: Check out repository code
16+
uses: actions/checkout@v3
17+
- name: Configure gcloud container and activate service account
18+
uses: ./.github/actions/configure-gcloud-container
19+
with:
20+
vault_secret_path: ${{ env.VAULT_SECRET_PATH }}
21+
vault_addr: ${{ secrets.VAULT_ADDR }}
22+
vault_role_id: ${{ secrets.VAULT_ROLE_ID }}
23+
vault_secret_id: ${{ secrets.VAULT_SECRET_ID }}
24+
- name: Authenticate to docker
25+
shell: bash
26+
run: |
27+
# auth into GCR via Docker with token from service account
28+
AUTH_TOKEN=$(docker run --rm ${{ env.IMAGE_NAME }} gcloud auth print-access-token)
29+
# need separate logins for gcr.io & marketplace.gcr.io
30+
echo $AUTH_TOKEN | docker login -u oauth2accesstoken --password-stdin https://gcr.io
31+
echo $AUTH_TOKEN | docker login -u oauth2accesstoken --password-stdin https://marketplace.gcr.io
32+
- name: Build and publish base Docker image
33+
id: build-and-publish
34+
run: |
35+
set +e # continue to allow capturing exit code
36+
ci/gha.publish-image.bash
37+
echo "exit-code=$?" >> $GITHUB_OUTPUT
38+
- name: Set exit code
39+
uses: ./.github/actions/set-exit-code
40+
with:
41+
exit-code: ${{ steps.build-and-publish.outputs.exit-code }}
42+

‎.github/workflows/test-base-docker-image.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Test rails-baseimage Docker image
2+
on:
3+
push:
4+
branches-ignore:
5+
- master
6+
workflow_dispatch:
7+
pull_request:
8+
env:
9+
VAULT_SECRET_PATH: 'secret/kdux/scp/staging/scp_service_account.json'
10+
IMAGE_NAME: 'gcloud-config'
11+
12+
jobs:
13+
Build-And-Test-Docker-Image:
14+
runs-on: ubuntu-20.04
15+
steps:
16+
- name: Check out repository code
17+
uses: actions/checkout@v3
18+
- name: Configure gcloud container and activate service account
19+
uses: ./.github/actions/configure-gcloud-container
20+
with:
21+
vault_secret_path: ${{ env.VAULT_SECRET_PATH }}
22+
vault_addr: ${{ secrets.VAULT_ADDR }}
23+
vault_role_id: ${{ secrets.VAULT_ROLE_ID }}
24+
vault_secret_id: ${{ secrets.VAULT_SECRET_ID }}
25+
- name: Authenticate to GCR
26+
shell: bash
27+
run: |
28+
# auth into GCR via Docker with token from service account
29+
AUTH_TOKEN=$(docker run --rm ${{ env.IMAGE_NAME }} gcloud auth print-access-token)
30+
# need separate logins for gcr.io & marketplace.gcr.io
31+
echo $AUTH_TOKEN | docker login -u oauth2accesstoken --password-stdin https://gcr.io
32+
echo $AUTH_TOKEN | docker login -u oauth2accesstoken --password-stdin https://marketplace.gcr.io
33+
- name: Test candidate base Docker image
34+
id: build-and-test
35+
run: |
36+
set +e # continue to allow capturing exit code
37+
ci/gha.build-and-test.bash
38+
echo "exit-code=$?" >> $GITHUB_OUTPUT
39+
- name: Set exit code
40+
uses: ./.github/actions/set-exit-code
41+
with:
42+
exit-code: ${{ steps.build-and-test.outputs.exit-code }}

‎ci/gha-build

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
3+
# USAGE: run with no arguments on a system with all prequisites installed to build the image. See also: local_build
4+
# TODO TODO: maybe commit some todo's and grep for them at the end of the test job?
5+
6+
THIS_DIR="$(cd "$(dirname "$BASH_SOURCE")"; pwd)"
7+
BASE_DIR="$(dirname $THIS_DIR)"
8+
BIN="$BASE_DIR/lib/bin"
9+
BASH_INCLUDE="$BASE_DIR/lib/bash_include"
10+
11+
. $BASH_INCLUDE/show_banner.bash
12+
. $BASH_INCLUDE/bash_utils.bash
13+
. $BASH_INCLUDE/dockerhub.bash
14+
. $BASH_INCLUDE/extract_vault_secrets.bash
15+
16+
function main {
17+
18+
force_pull_base_image || exit_with_error_message "FAILED to force pull base image."
19+
20+
$BIN/rebuild_phusion_baseimage || exit_with_error_message "FAILED to rebuild phusion base image"
21+
$BIN/rebuild_phusion_passenger_image || exit_with_error_message "FAILED to rebuild phusion passenger image"
22+
$BIN/rebuild_image || exit_with_error_message "FAILED to build docker image"
23+
}
24+
25+
# Forcing the true latest version of $BASE_IMAGE .
26+
function force_pull_base_image {
27+
echo
28+
echo '########################################'
29+
echo "## pulling fresh BASE_IMAGE $BASE_IMAGE"
30+
echo '########################################'
31+
32+
docker_force_pull $BASE_IMAGE || exit_with_error_message "FAILED to pull base image $BASE_IMAGE ."
33+
echo "showing image history for \"$BASE_IMAGE\":"
34+
docker history -H=false --no-trunc $BASE_IMAGE
35+
echo "showing image inspect for \"$BASE_IMAGE\":"
36+
docker inspect $BASE_IMAGE
37+
}
38+
39+
main "$@"

‎ci/gha-publish

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
3+
# USAGE: run this with no argumants to publish the image that was built. Or better yet, don't run this, and let the jenkins job take care of it.
4+
5+
THIS_DIR="$(cd "$(dirname "$BASH_SOURCE")"; pwd)"
6+
BASE_DIR="$(dirname $THIS_DIR)"
7+
BASH_INCLUDE="$BASE_DIR/lib/bash_include"
8+
9+
. $BASH_INCLUDE/show_banner.bash
10+
. $BASH_INCLUDE/config.bash
11+
. $BASH_INCLUDE/dockerhub.bash
12+
13+
function main {
14+
assert_main_version_unused
15+
16+
for PUBLISH_VERSION in "$MAIN_VERSION" "latest";do
17+
docker tag "$MAIN_IMAGE_FULLNAME:candidate" "$MAIN_GCR_IMAGE_FULLNAME:$PUBLISH_VERSION" || exit_with_error_message "FAILED to tag $MAIN_GCR_IMAGE_FULLNAME with $PUBLISH_VERSION "
18+
docker image ls | grep "\\b$MAIN_GCR_IMAGE_FULLNAME\\b" | grep "\\b$PUBLISH_VERSION\\b" || exit 1 #confirm taggging worked
19+
docker push "$MAIN_GCR_IMAGE_FULLNAME:$PUBLISH_VERSION" || exit_with_error_message "FAILED to push $MAIN_GCR_IMAGE_FULLNAME:$PUBLISH_VERSION " # TODO: check if it FAILED because of docker login being needed
20+
echo "Published $MAIN_GCR_IMAGE_FULLNAME:$PUBLISH_VERSION -- see it at https://console.cloud.google.com/gcr/images/$GCR_PROJECT/GLOBAL/$MAIN_IMAGE_NAME?project=$GCR_PROJECT"
21+
done
22+
}
23+
24+
main "$@"

‎ci/gha.build-and-test.bash

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
3+
# USAGE: Use without arguments in a jenkins test job to build and test the image. This keeps jenkins job config complexity low.
4+
5+
THIS_DIR="$(cd "$(dirname "$BASH_SOURCE")"; pwd)"
6+
BASE_DIR="$(dirname $THIS_DIR)"
7+
BASH_INCLUDE="$BASE_DIR/lib/bash_include"
8+
9+
. $BASH_INCLUDE/show_banner.bash
10+
. $BASH_INCLUDE/bash_utils.bash
11+
12+
$THIS_DIR/gha-build || exit_with_error_message "build FAILED"
13+
$THIS_DIR/test || exit_with_error_message "test FAILED"
14+
$THIS_DIR/check_for_newer_dependencies || { echo "WARNING: Setting jenkins job to unstable to trigger email for new dependency versions that this build has not adopted yet..." >&2; exit 99; }

‎ci/gha.publish-image.bash

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
# USAGE: Use without arguments in a jenkins test job to build and test the image. This keeps jenkins job config complexity low.
4+
5+
THIS_DIR="$(cd "$(dirname "$BASH_SOURCE")"; pwd)"
6+
BASE_DIR="$(dirname $THIS_DIR)"
7+
BASH_INCLUDE="$BASE_DIR/lib/bash_include"
8+
9+
. $BASH_INCLUDE/show_banner.bash
10+
. $BASH_INCLUDE/bash_utils.bash
11+
. $BASE_DIR/lib/bash_include/extract_vault_secrets.bash
12+
13+
$THIS_DIR/gha-build && $THIS_DIR/test && $THIS_DIR/gha-publish || exit 1

‎version.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.2.1
1+
2.2.2

0 commit comments

Comments
 (0)