Skip to content

Commit ea94994

Browse files
committed
cicd: initial upstream pipeline
1 parent 2660ea6 commit ea94994

11 files changed

Lines changed: 323 additions & 21 deletions

.gitlab-ci.yml

Lines changed: 82 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,110 @@
11
stages:
22
- init
3-
- test
3+
- run
44
- finish
55

6+
.base:
7+
variables:
8+
ARCHES: "x86_64"
9+
TYPES: "qcow2"
10+
FEDORA_VERSION: "43"
11+
EL9_VERSION: "9.7"
12+
EL10_VERSION: "10.1"
13+
FROM_REFS: |-
14+
quay.io/fedora/fedora-bootc:${FEDORA_VERSION}
15+
quay.io/centos-bootc/centos-bootc:stream10
16+
quay.io/centos-bootc/centos-bootc:stream9
17+
registry.redhat.io/rhel9/rhel-bootc:${EL9_VERSION}
18+
registry.redhat.io/rhel10/rhel-bootc:${EL10_VERSION}
19+
620
init:
21+
extends: .base
722
stage: init
823
interruptible: true
924
tags:
1025
- shell
1126
script:
12-
- schutzbot/update_github_status.sh start
27+
- |
28+
for FROM_REF in $FROM_REFS; do
29+
schutzbot/update_github_status.sh start "$FROM_REF"
30+
done
31+
32+
run:
33+
extends: .base
34+
stage: run
35+
parallel:
36+
matrix:
37+
- FROM_REF: "quay.io/fedora/fedora-bootc:${FEDORA_VERSION}"
38+
CONTAINERFILE: "fedora"
39+
RUNNER: "aws/centos-stream-10-x86_64"
40+
FROM_CREDS: ""
41+
DST_REF: "quay.io/osbuild/fedora-bootc:${FEDORA_VERSION}"
42+
DST_CREDS: "$QUAY_CREDS"
43+
44+
- FROM_REF: "registry.redhat.io/rhel9/rhel-bootc:${EL9_VERSION}"
45+
CONTAINERFILE: "el9"
46+
RUNNER: "aws/rhel-10.1-ga-x86_64"
47+
FROM_CREDS: "$RH_CREDS"
48+
DST_REF: ""
49+
DST_CREDS: ""
50+
51+
- FROM_REF: "quay.io/centos-bootc/centos-bootc:stream10"
52+
CONTAINERFILE: "el10"
53+
RUNNER: "aws/centos-stream-10-x86_64"
54+
FROM_CREDS: ""
55+
DST_REF: "quay.io/osbuild/centos-bootc:stream10"
56+
DST_CREDS: "$QUAY_CREDS"
57+
58+
- FROM_REF: "quay.io/centos-bootc/centos-bootc:stream9"
59+
CONTAINERFILE: "el9"
60+
RUNNER: "aws/centos-stream-10-x86_64"
61+
FROM_CREDS: ""
62+
DST_REF: "quay.io/osbuild/centos-bootc:stream9"
63+
DST_CREDS: "$QUAY_CREDS"
64+
65+
- FROM_REF: "registry.redhat.io/rhel10/rhel-bootc:${EL10_VERSION}"
66+
CONTAINERFILE: "el10"
67+
RUNNER: "aws/rhel-10.1-ga-x86_64"
68+
FROM_CREDS: "$RH_CREDS"
69+
DST_REF: ""
70+
DST_CREDS: ""
1371

14-
test:
1572
before_script:
16-
- mkdir -p /tmp/artifacts
17-
- schutzbot/ci_details.sh > /tmp/artifacts/ci-details-before-run.txt
73+
- . schutzbot/gitlab-utils.sh
74+
- section_start before_script "Run before script section"
75+
- schutzbot/ec2-spot-check.sh &
76+
- mkdir -p /tmp/artifacts && schutzbot/ci_details.sh > /tmp/artifacts/ci-details-before-run.txt
1877
- cat schutzbot/team_ssh_keys.txt | tee -a ~/.ssh/authorized_keys > /dev/null
78+
- sudo dnf -y install buildah
79+
- |
80+
# Push only on main branch
81+
if [ "$CI_COMMIT_BRANCH" != "main" ]; then
82+
unset DST_CREDS
83+
fi
84+
- section_end before_script
1985
script:
20-
- schutzbot/noop.sh
86+
- schutzbot/build.sh
2187
after_script:
88+
- . schutzbot/gitlab-utils.sh
89+
- section_start after_script "Run after script section"
2290
- schutzbot/ci_details.sh > /tmp/artifacts/ci-details-after-run.txt || true
2391
- schutzbot/unregister.sh || true
24-
- schutzbot/update_github_status.sh update || true
92+
- schutzbot/update_github_status.sh update "$FROM_REF" || true
2593
- schutzbot/save_journal.sh || true
2694
- schutzbot/upload_artifacts.sh
95+
- if [ "$CI_JOB_STATUS" = "success" ]; then schutzbot/update_github_status.sh finish "$FROM_REF"; fi
96+
- section_end after_script
2797
tags:
2898
- terraform/openstack
29-
parallel:
30-
matrix:
31-
# available runners: https://github.com/osbuild/gitlab-ci-terraform
32-
- RUNNER:
33-
- aws/centos-stream-10-x86_64
34-
- aws/rhel-10.1-ga-x86_64
35-
INTERNAL_NETWORK: ["true"]
3699

37100
finish:
101+
extends: .base
38102
stage: finish
39103
dependencies: []
40104
tags:
41105
- shell
42106
script:
43-
- schutzbot/update_github_status.sh finish
107+
- |
108+
for FROM_REF in $FROM_REFS; do
109+
schutzbot/update_github_status.sh finish "$FROM_REF"
110+
done

Containerfile.comment

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#
2+
# TODO: This comment will be replaced with explanation on how to use the Containerfile
3+
# to rebuild a bootable container deployment once we get to testing those images.
4+
#
5+
# https://github.com/osbuild/bootc-foundry/issues/9
6+
#

Containerfile.el10-qcow2

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
FROM quay.io/centos-bootc/centos-bootc
2+
3+
RUN echo TODO

Containerfile.el9-qcow2

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
FROM quay.io/centos-bootc/centos-bootc
2+
3+
RUN echo TODO

Containerfile.fedora-qcow2

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
FROM quay.io/fedora/fedora-bootc
2+
3+
RUN echo TODO

schutzbot/README.md

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Schutzbot
2+
3+
This directory contains GitHub -> GitLab CICD pipeline.
4+
5+
## The workflow:
6+
7+
* GitHub Action `trigger-gitlab.yml` checkouts the branch and pushes it into GitLab.
8+
* GitLab starts CICD pipeline base on `.gitlab-ci.yml`.
9+
* Terraform is checked out (`schutzbot/terraform`) and deploys runner instances.
10+
* Terraform Executor prepares the runners.
11+
* Script `schutzbot/update_github_status.sh` sets GitHub state to RUNNING.
12+
* Runners executes scripts from `schutzbot/`
13+
* Script `schutzbot/update_github_status.sh` updates GitHub state (PASS/FAIL).
14+
15+
## GitLab Repo
16+
17+
https://gitlab.com/redhat/services/products/image-builder/ci/osbuild-foundry
18+
19+
## Creating a new setup
20+
21+
* Create GitHub/GitLab repositories.
22+
* Make sure GitLab project that is *public*.
23+
* Create `.github/trigger-gitlab.yml`.
24+
* Set required GitHub Action variables.
25+
* Create `.gitlab-ci.yml`.
26+
* Set required GitLab CICD variables.
27+
28+
## Other links
29+
30+
* https://github.com/osbuild/image-builder-terraform
31+
* https://github.com/osbuild/gitlab-ci-config
32+
* https://github.com/osbuild/gitlab-ci-terraform
33+
* https://github.com/osbuild/gitlab-ci-terraform-executor

schutzbot/build.sh

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/usr/bin/env bash
2+
set -xeuo pipefail
3+
4+
#
5+
# Builds and creates multi-arch manifest; optionally pushes. Required variables:
6+
#
7+
# FROM_REF - base ref (e.g. quay.io/fedora/fedora-bootc:43)
8+
# DST_REF - destination ref image without image type suffix (e.g. quay.io/osbuild/fedora-bootc:43)
9+
# CONTAINERFILE - containerfile suffix without type (e.g. fedora → Containerfile.fedora-xxx)
10+
#
11+
# Optional variables:
12+
#
13+
# ARCHES, TYPES - space-separated lists (default from .gitlab-ci.yml: x86_64, qcow2)
14+
# FROM_CREDS - colon-separated username:password for source registry (none for public registries)
15+
# DST_CREDS - colon-separated username:password for destination registry (skip push if missing)
16+
# CACHE_REF, CACHE_USERNAME, CACHE_PASSWORD - sidecar cache registry (optional)
17+
18+
. "$(dirname "$0")/gitlab-utils.sh"
19+
20+
# Parse colon-separated username:password (password may contain colons)
21+
if [ -n "${FROM_CREDS:-}" ] && [[ "${FROM_CREDS}" == *:* ]]; then
22+
FROM_USER="${FROM_CREDS%%:*}"
23+
FROM_PASS="${FROM_CREDS#*:}"
24+
section_start login_from_registry "Logging in to $FROM_REF"
25+
TO_REGISTRY=$(echo "$FROM_REF" | cut -d/ -f1)
26+
echo "$FROM_PASS" | buildah login -u "$FROM_USER" --password-stdin "$TO_REGISTRY"
27+
section_end login_from_registry
28+
fi
29+
30+
if [ -n "${DST_CREDS:-}" ] && [[ "${DST_CREDS}" == *:* ]]; then
31+
DST_USER="${DST_CREDS%%:*}"
32+
DST_PASS="${DST_CREDS#*:}"
33+
section_start login_dst_registry "Logging in to $DST_REF"
34+
TO_REGISTRY=$(echo "$DST_REF" | cut -d/ -f1)
35+
echo "$DST_PASS" | buildah login -u "$DST_USER" --password-stdin "$TO_REGISTRY"
36+
section_end login_dst_registry
37+
fi
38+
39+
if [ -n "${CACHE_REF:-}" ] && [ -n "${CACHE_USERNAME:-}" ] && [ -n "${CACHE_PASSWORD:-}" ]; then
40+
section_start login_cache_ref "Logging in to $CACHE_REF (cache will be used)"
41+
CACHE_REGISTRY=$(echo "$CACHE_REF" | cut -d/ -f1)
42+
echo "$CACHE_PASSWORD" | buildah login -u "$CACHE_USERNAME" --password-stdin "$CACHE_REGISTRY"
43+
section_end login_cache_ref
44+
45+
section_start pull_cache "Pulling from cache ref ${CACHE_REF}"
46+
echo "Using cache $CACHE_REF"
47+
buildah pull "$CACHE_REF" || true
48+
buildah pull "$FROM_REF"
49+
section_end pull_cache
50+
51+
section_start push_to_cache "Pushing to cache ref ${CACHE_REF}"
52+
buildah push "$FROM_REF" "$CACHE_REF" || true
53+
section_end push_to_cache
54+
fi
55+
56+
NOPUSH=
57+
if [ -z "${DST_REF:-}" ]; then
58+
DST_REF="local"
59+
NOPUSH=1
60+
fi
61+
62+
ARCHES=${ARCHES:-x86_64}
63+
TYPES=${TYPES:-qcow2}
64+
DST_REF=${DST_REF:-local}
65+
66+
for ARCH in $ARCHES; do
67+
for TYPE in $TYPES; do
68+
section_start prepare_containerfile "Preparing Containerfile with FROM ${FROM_REF}"
69+
cp "Containerfile.comment" "Containerfile"
70+
{
71+
echo "FROM ${FROM_REF}"
72+
echo "ARG BUILD_DATE=$(date +%Y-%m-%d)"
73+
tail -n +2 "Containerfile.${CONTAINERFILE}-${TYPE}"
74+
} >> "Containerfile"
75+
section_end prepare_containerfile
76+
77+
section_start "build_${ARCH}_${TYPE}" "Building $FROM_REF arch $ARCH type $TYPE"
78+
buildah build --layers --arch="$ARCH" \
79+
--build-arg CONTAINERFILE="Containerfile" \
80+
--from "${FROM_REF}" \
81+
-f "Containerfile" \
82+
-t "$DST_REF-$ARCH-$TYPE" .
83+
section_end "build_${ARCH}_${TYPE}"
84+
done
85+
done
86+
87+
for TYPE in $TYPES; do
88+
for ARCH in $ARCHES; do
89+
section_start "push_${ARCH}_${TYPE}" "Pushing $DST_REF-$ARCH-$TYPE to registry"
90+
if [ -n "${DST_CREDS:-}" ] && [ -z "${NOPUSH:-}" ]; then
91+
echo "Pushing $DST_REF-$ARCH-$TYPE to registry"
92+
buildah push "$DST_REF-$ARCH-$TYPE"
93+
else
94+
echo "Push skipped: DST_CREDS missing or NOPUSH set"
95+
fi
96+
section_end "push_${ARCH}_${TYPE}"
97+
done
98+
done
99+
100+
section_start create_manifest "Creating manifest for $DST_REF"
101+
buildah manifest create "$DST_REF"
102+
section_end create_manifest
103+
104+
for TYPE in $TYPES; do
105+
section_start "add_to_manifest_${TYPE}" "Adding $TYPE to $DST_REF manifest"
106+
for ARCH in $ARCHES; do
107+
buildah manifest add "$DST_REF" "$DST_REF-$ARCH-$TYPE"
108+
done
109+
section_end "add_to_manifest_${TYPE}"
110+
111+
if [ -n "${DST_CREDS:-}" ] && [ -z "${NOPUSH:-}" ]; then
112+
section_start "push_manifest_${TYPE}" "Pushing manifest $DST_REF to registry"
113+
buildah manifest push --all "$DST_REF" "docker://$DST_REF"
114+
section_end "push_manifest_${TYPE}"
115+
else
116+
echo "Push skipped: DST_CREDS missing or NOPUSH set"
117+
fi
118+
done

schutzbot/ec2-spot-check.sh

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
3+
# Run this script on the background to shutdown git runner gracefully on
4+
# AWS EC2 spot termination. Does nothing for non-EC2 environments.
5+
6+
if [ -f /sys/class/dmi/id/product_uuid ]; then
7+
UUID=$(tr '[:upper:]' '[:lower:]' < /sys/class/dmi/id/product_uuid)
8+
if [[ "$UUID" != ec2* ]]; then
9+
exit 0
10+
fi
11+
else
12+
exit 0
13+
fi
14+
15+
while true; do
16+
TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
17+
HTTP_CODE=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s -w "%{http_code}" -o /dev/null http://169.254.169.254/latest/meta-data/spot/instance-action)
18+
19+
if [ "$HTTP_CODE" -eq 200 ]; then
20+
MSG="EC2 Spot Interruption detected! Shutting down GitLab Runner gracefully."
21+
echo "$MSG"
22+
logger -t spot-monitor "$MSG"
23+
BODY=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/spot/instance-action)
24+
echo "$BODY"
25+
logger -t spot-monitor "$BODY"
26+
27+
schutzbot/update_github_status.sh spot
28+
gitlab-runner stop
29+
break
30+
fi
31+
32+
sleep 30
33+
done

schutzbot/gitlab-utils.sh

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Prints a section start message to the console.
5+
#
6+
# Arguments:
7+
# $1 - section id
8+
# $2 - section title
9+
# $3 - collapsed (optional)
10+
#
11+
# Example:
12+
# section_start build_image "Building image"
13+
function section_start() {
14+
set +x
15+
local section_id="${1}_$$"
16+
local section_title=$2
17+
local collapsed=${3:-true}
18+
local params=""
19+
[ "$collapsed" == "true" ] && params="[collapsed=true]"
20+
21+
printf "\e[0Ksection_start:%s:%s%s\r\e[0K\e[1;36m%s\e[0m\n" "$(date +%s)" "$section_id" "$params" "$section_title"
22+
set -x
23+
}
24+
25+
# Prints a section end message to the console.
26+
#
27+
# Arguments:
28+
# $1 - section id
29+
#
30+
# Example:
31+
# section_end build_image
32+
function section_end() {
33+
set +x
34+
local section_id="${1}_$$"
35+
printf "\e[0Ksection_end:%s:%s\r\e[0K\n" "$(date +%s)" "$section_id"
36+
set -x
37+
}

schutzbot/noop.sh

Lines changed: 0 additions & 4 deletions
This file was deleted.

0 commit comments

Comments
 (0)