Skip to content

Commit a945c32

Browse files
committed
cicd: initial upstream pipeline
1 parent 2660ea6 commit a945c32

11 files changed

Lines changed: 318 additions & 21 deletions

.gitlab-ci.yml

Lines changed: 77 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,105 @@
11
stages:
22
- init
3-
- test
3+
- run
44
- finish
55

6+
.base:
7+
variables:
8+
ARCHES: "x86_64"
9+
TYPES: "qcow2"
10+
FEDORA_VERSION: "43"
11+
EL9_VERSION: "9.7"
12+
EL10_VERSION: "10.1"
13+
FROM_REFS: "quay.io/fedora/fedora-bootc:${FEDORA_VERSION} registry.redhat.io/rhel9/rhel-bootc:${EL9_VERSION} registry.redhat.io/rhel10/rhel-bootc:${EL10_VERSION}"
14+
615
init:
16+
extends: .base
717
stage: init
818
interruptible: true
919
tags:
1020
- shell
1121
script:
12-
- schutzbot/update_github_status.sh start
22+
- |
23+
for FROM_REF in $FROM_REFS; do
24+
schutzbot/update_github_status.sh start "$FROM_REF"
25+
done
26+
27+
run:
28+
extends: .base
29+
stage: run
30+
parallel:
31+
matrix:
32+
- FROM_REF: "quay.io/fedora/fedora-bootc:${FEDORA_VERSION}"
33+
CONTAINERFILE: "fedora"
34+
RUNNER: "aws/centos-stream-10-x86_64"
35+
FROM_CREDS: ""
36+
DST_REF: "quay.io/osbuild/fedora-bootc:${FEDORA_VERSION}"
37+
DST_CREDS: "$QUAY_CREDS"
38+
39+
- FROM_REF: "registry.redhat.io/rhel9/rhel-bootc:${EL9_VERSION}"
40+
CONTAINERFILE: "el9"
41+
RUNNER: "aws/rhel-10.1-ga-x86_64"
42+
FROM_CREDS: "$RH_CREDS"
43+
DST_REF: ""
44+
DST_CREDS: ""
45+
46+
- FROM_REF: "quay.io/centos-bootc/centos-bootc:stream10"
47+
CONTAINERFILE: "el10"
48+
RUNNER: "aws/centos-stream-10-x86_64"
49+
FROM_CREDS: ""
50+
DST_REF: "quay.io/osbuild/centos-bootc:stream10"
51+
DST_CREDS: "$QUAY_CREDS"
52+
53+
- FROM_REF: "quay.io/centos-bootc/centos-bootc:stream9"
54+
CONTAINERFILE: "el9"
55+
RUNNER: "aws/centos-stream-10-x86_64"
56+
FROM_CREDS: ""
57+
DST_REF: "quay.io/osbuild/centos-bootc:stream9"
58+
DST_CREDS: "$QUAY_CREDS"
59+
60+
- FROM_REF: "registry.redhat.io/rhel10/rhel-bootc:${EL10_VERSION}"
61+
CONTAINERFILE: "el10"
62+
RUNNER: "aws/rhel-10.1-ga-x86_64"
63+
FROM_CREDS: "$RH_CREDS"
64+
DST_REF: ""
65+
DST_CREDS: ""
1366

14-
test:
1567
before_script:
16-
- mkdir -p /tmp/artifacts
17-
- schutzbot/ci_details.sh > /tmp/artifacts/ci-details-before-run.txt
68+
- . schutzbot/gitlab-utils.sh
69+
- section_start before_script "Run before script section"
70+
- schutzbot/ec2-spot-check.sh &
71+
- mkdir -p /tmp/artifacts && schutzbot/ci_details.sh > /tmp/artifacts/ci-details-before-run.txt
1872
- cat schutzbot/team_ssh_keys.txt | tee -a ~/.ssh/authorized_keys > /dev/null
73+
- sudo dnf -y install buildah
74+
- |
75+
# Push only on main branch
76+
if [ "$CI_COMMIT_BRANCH" != "main" ]; then
77+
unset DST_CREDS
78+
fi
79+
- section_end before_script
1980
script:
20-
- schutzbot/noop.sh
81+
- schutzbot/build.sh
2182
after_script:
83+
- . schutzbot/gitlab-utils.sh
84+
- section_start after_script "Run after script section"
2285
- schutzbot/ci_details.sh > /tmp/artifacts/ci-details-after-run.txt || true
2386
- schutzbot/unregister.sh || true
24-
- schutzbot/update_github_status.sh update || true
87+
- schutzbot/update_github_status.sh update "$FROM_REF" || true
2588
- schutzbot/save_journal.sh || true
2689
- schutzbot/upload_artifacts.sh
90+
- if [ "$CI_JOB_STATUS" = "success" ]; then schutzbot/update_github_status.sh finish "$FROM_REF"; fi
91+
- section_end after_script
2792
tags:
2893
- terraform/openstack
29-
parallel:
30-
matrix:
31-
# available runners: https://github.com/osbuild/gitlab-ci-terraform
32-
- RUNNER:
33-
- aws/centos-stream-10-x86_64
34-
- aws/rhel-10.1-ga-x86_64
35-
INTERNAL_NETWORK: ["true"]
3694

3795
finish:
96+
extends: .base
3897
stage: finish
3998
dependencies: []
4099
tags:
41100
- shell
42101
script:
43-
- schutzbot/update_github_status.sh finish
102+
- |
103+
for FROM_REF in $FROM_REFS; do
104+
schutzbot/update_github_status.sh finish "$FROM_REF"
105+
done

Containerfile.comment

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#
2+
# TODO: This comment will be replaced with explanation on how to use the Containerfile
3+
# to rebuild a bootable container deployment once we get to testing those images.
4+
#
5+
# https://github.com/osbuild/bootc-foundry/issues/9
6+
#

Containerfile.el10-qcow2

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
FROM quay.io/centos-bootc/centos-bootc
2+
3+
RUN echo TODO

Containerfile.el9-qcow2

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
FROM quay.io/centos-bootc/centos-bootc
2+
3+
RUN echo TODO

Containerfile.fedora-qcow2

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
FROM quay.io/fedora/fedora-bootc
2+
3+
RUN echo TODO

schutzbot/README.md

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Schutzbot
2+
3+
This directory contains GitHub -> GitLab CICD pipeline.
4+
5+
## The workflow:
6+
7+
* GitHub Action `trigger-gitlab.yml` checkouts the branch and pushes it into GitLab.
8+
* GitLab starts CICD pipeline base on `.gitlab-ci.yml`.
9+
* Terraform is checked out (`schutzbot/terraform`) and deploys runner instances.
10+
* Terraform Executor prepares the runners.
11+
* Script `schutzbot/update_github_status.sh` sets GitHub state to RUNNING.
12+
* Runners executes scripts from `schutzbot/`
13+
* Script `schutzbot/update_github_status.sh` updates GitHub state (PASS/FAIL).
14+
15+
## GitLab Repo
16+
17+
https://gitlab.com/redhat/services/products/image-builder/ci/osbuild-foundry
18+
19+
## Creating a new setup
20+
21+
* Create GitHub/GitLab repositories.
22+
* Make sure GitLab project that is *public*.
23+
* Create `.github/trigger-gitlab.yml`.
24+
* Set required GitHub Action variables.
25+
* Create `.gitlab-ci.yml`.
26+
* Set required GitLab CICD variables.
27+
28+
## Other links
29+
30+
* https://github.com/osbuild/image-builder-terraform
31+
* https://github.com/osbuild/gitlab-ci-config
32+
* https://github.com/osbuild/gitlab-ci-terraform
33+
* https://github.com/osbuild/gitlab-ci-terraform-executor

schutzbot/build.sh

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/usr/bin/env bash
2+
set -xeuo pipefail
3+
4+
#
5+
# Builds and creates multi-arch manifest; optionally pushes. Required variables:
6+
#
7+
# FROM_REF - base ref (e.g. quay.io/fedora/fedora-bootc:43)
8+
# DST_REF - destination ref image without image type suffix (e.g. quay.io/osbuild/fedora-bootc:43)
9+
# CONTAINERFILE - containerfile suffix without type (e.g. fedora → Containerfile.fedora-xxx)
10+
#
11+
# Optional variables:
12+
#
13+
# ARCHES, TYPES - space-separated lists (default from .gitlab-ci.yml: x86_64, qcow2)
14+
# FROM_CREDS - colon-separated username:password for source registry (none for public registries)
15+
# DST_CREDS - colon-separated username:password for destination registry (skip push if missing)
16+
# CACHE_REF, CACHE_USERNAME, CACHE_PASSWORD - sidecar cache registry (optional)
17+
18+
. "$(dirname "$0")/gitlab-utils.sh"
19+
20+
# Parse colon-separated username:password (password may contain colons)
21+
if [ -n "${FROM_CREDS:-}" ] && [[ "${FROM_CREDS}" == *:* ]]; then
22+
FROM_USER="${FROM_CREDS%%:*}"
23+
FROM_PASS="${FROM_CREDS#*:}"
24+
section_start login_from_registry "Logging in to $FROM_REF"
25+
TO_REGISTRY=$(echo "$FROM_REF" | cut -d/ -f1)
26+
echo "$FROM_PASS" | buildah login -u "$FROM_USER" --password-stdin "$TO_REGISTRY"
27+
section_end login_from_registry
28+
fi
29+
30+
if [ -n "${DST_CREDS:-}" ] && [[ "${DST_CREDS}" == *:* ]]; then
31+
DST_USER="${DST_CREDS%%:*}"
32+
DST_PASS="${DST_CREDS#*:}"
33+
section_start login_dst_registry "Logging in to $DST_REF"
34+
TO_REGISTRY=$(echo "$DST_REF" | cut -d/ -f1)
35+
echo "$DST_PASS" | buildah login -u "$DST_USER" --password-stdin "$TO_REGISTRY"
36+
section_end login_dst_registry
37+
fi
38+
39+
if [ -n "${CACHE_REF:-}" ] && [ -n "${CACHE_USERNAME:-}" ] && [ -n "${CACHE_PASSWORD:-}" ]; then
40+
section_start login_cache_ref "Logging in to $CACHE_REF (cache will be used)"
41+
CACHE_REGISTRY=$(echo "$CACHE_REF" | cut -d/ -f1)
42+
echo "$CACHE_PASSWORD" | buildah login -u "$CACHE_USERNAME" --password-stdin "$CACHE_REGISTRY"
43+
section_end login_cache_ref
44+
45+
section_start pull_cache "Pulling from cache ref ${CACHE_REF}"
46+
echo "Using cache $CACHE_REF"
47+
buildah pull "$CACHE_REF" || true
48+
buildah pull "$FROM_REF"
49+
section_end pull_cache
50+
51+
section_start push_to_cache "Pushing to cache ref ${CACHE_REF}"
52+
buildah push "$FROM_REF" "$CACHE_REF" || true
53+
section_end push_to_cache
54+
fi
55+
56+
NOPUSH=
57+
if [ -z "${DST_REF:-}" ]; then
58+
DST_REF="local"
59+
NOPUSH=1
60+
fi
61+
62+
ARCHES=${ARCHES:-x86_64}
63+
TYPES=${TYPES:-qcow2}
64+
DST_REF=${DST_REF:-local}
65+
66+
for ARCH in $ARCHES; do
67+
for TYPE in $TYPES; do
68+
section_start prepare_containerfile "Preparing Containerfile with FROM ${FROM_REF}"
69+
cp "Containerfile.comment" "Containerfile"
70+
{
71+
echo "FROM ${FROM_REF}"
72+
echo "ARG BUILD_DATE=$(date +%Y-%m-%d)"
73+
tail -n +2 "Containerfile.${CONTAINERFILE}-${TYPE}"
74+
} >> "Containerfile"
75+
section_end prepare_containerfile
76+
77+
section_start "build_${ARCH}_${TYPE}" "Building $FROM_REF arch $ARCH type $TYPE"
78+
buildah build --layers --arch="$ARCH" \
79+
--build-arg CONTAINERFILE="Containerfile" \
80+
--from "${FROM_REF}" \
81+
-f "Containerfile" \
82+
-t "$DST_REF-$ARCH-$TYPE" .
83+
section_end "build_${ARCH}_${TYPE}"
84+
done
85+
done
86+
87+
for TYPE in $TYPES; do
88+
for ARCH in $ARCHES; do
89+
section_start "push_${ARCH}_${TYPE}" "Pushing $DST_REF-$ARCH-$TYPE to registry"
90+
if [ -n "${DST_CREDS:-}" ] && [ -z "${NOPUSH:-}" ]; then
91+
echo "Pushing $DST_REF-$ARCH-$TYPE to registry"
92+
buildah push "$DST_REF-$ARCH-$TYPE"
93+
else
94+
echo "Push skipped: DST_CREDS missing or NOPUSH set"
95+
fi
96+
section_end "push_${ARCH}_${TYPE}"
97+
done
98+
done
99+
100+
section_start create_manifest "Creating manifest for $DST_REF"
101+
buildah manifest create "$DST_REF"
102+
section_end create_manifest
103+
104+
for TYPE in $TYPES; do
105+
section_start "add_to_manifest_${TYPE}" "Adding $TYPE to $DST_REF manifest"
106+
for ARCH in $ARCHES; do
107+
buildah manifest add "$DST_REF" "$DST_REF-$ARCH-$TYPE"
108+
done
109+
section_end "add_to_manifest_${TYPE}"
110+
111+
if [ -n "${DST_CREDS:-}" ] && [ -z "${NOPUSH:-}" ]; then
112+
section_start "push_manifest_${TYPE}" "Pushing manifest $DST_REF to registry"
113+
buildah manifest push --all "$DST_REF" "docker://$DST_REF"
114+
section_end "push_manifest_${TYPE}"
115+
else
116+
echo "Push skipped: DST_CREDS missing or NOPUSH set"
117+
fi
118+
done

schutzbot/ec2-spot-check.sh

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
3+
# Run this script on the background to shutdown git runner gracefully on
4+
# AWS EC2 spot termination. Does nothing for non-EC2 environments.
5+
6+
if [ -f /sys/class/dmi/id/product_uuid ]; then
7+
UUID=$(tr '[:upper:]' '[:lower:]' < /sys/class/dmi/id/product_uuid)
8+
if [[ "$UUID" != ec2* ]]; then
9+
exit 0
10+
fi
11+
else
12+
exit 0
13+
fi
14+
15+
while true; do
16+
TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
17+
HTTP_CODE=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s -w "%{http_code}" -o /dev/null http://169.254.169.254/latest/meta-data/spot/instance-action)
18+
19+
if [ "$HTTP_CODE" -eq 200 ]; then
20+
MSG="EC2 Spot Interruption detected! Shutting down GitLab Runner gracefully."
21+
echo "$MSG"
22+
logger -t spot-monitor "$MSG"
23+
BODY=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" -s http://169.254.169.254/latest/meta-data/spot/instance-action)
24+
echo "$BODY"
25+
logger -t spot-monitor "$BODY"
26+
27+
schutzbot/update_github_status.sh spot
28+
gitlab-runner stop
29+
break
30+
fi
31+
32+
sleep 30
33+
done

schutzbot/gitlab-utils.sh

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
# Prints a section start message to the console.
5+
#
6+
# Arguments:
7+
# $1 - section id
8+
# $2 - section title
9+
# $3 - collapsed (optional)
10+
#
11+
# Example:
12+
# section_start build_image "Building image"
13+
function section_start() {
14+
set +x
15+
local section_id="${1}_$$"
16+
local section_title=$2
17+
local collapsed=${3:-true}
18+
local params=""
19+
[ "$collapsed" == "true" ] && params="[collapsed=true]"
20+
21+
printf "\e[0Ksection_start:%s:%s%s\r\e[0K\e[1;36m%s\e[0m\n" "$(date +%s)" "$section_id" "$params" "$section_title"
22+
set -x
23+
}
24+
25+
# Prints a section end message to the console.
26+
#
27+
# Arguments:
28+
# $1 - section id
29+
#
30+
# Example:
31+
# section_end build_image
32+
function section_end() {
33+
set +x
34+
local section_id="${1}_$$"
35+
printf "\e[0Ksection_end:%s:%s\r\e[0K\n" "$(date +%s)" "$section_id"
36+
set -x
37+
}

schutzbot/noop.sh

Lines changed: 0 additions & 4 deletions
This file was deleted.

0 commit comments

Comments
 (0)