Skip to content

Commit 915546d

Browse files
ahmedelkashevAlexey-N-Chernyshov
ahmedelkashev
and
Alexey-N-Chernyshov
authored
Feature/GitHub action tag and publish docker image (#626)
* GitHub action tag and publish docker image pipeline with github actions Signed-off-by: Ahmed Elkashef <[email protected]> Co-authored-by: Alexey <[email protected]>
1 parent de32696 commit 915546d

12 files changed

+536
-11
lines changed

‎.github/_README.md

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
GitHub Actions
2+
==============
3+
4+
GitHub Workflow description in YAML does not support anchors.
5+
There are several workarounds => anyway they come to building-editing workflow yaml from source.
6+
So I suggest yet another one `make-workflows.sh` based on YAML tool `yq`.
7+
8+
### USAGE
9+
0. Move your workflows to `.github/*.src.yml`
10+
1. Put `make-workflows.sh` to directory `.github/`
11+
2. (optional) Copy or link `pre-commit-hook.sh` to `.git/hooks/pre-commit`
12+
Like `ln -s ../../.github/pre-commit-hook.sh .git/hooks/pre-commit`
13+
14+
### Using pre-commit
15+
```yaml
16+
repos:
17+
- repo: local
18+
hooks:
19+
- id: make-workflows
20+
name: Make GitHub workflows from *.src.yml
21+
entry: bash -c '.github/make-workflows.sh && git add .github/workflows'
22+
language: system
23+
types: [yaml]
24+
pass_filenames: false
25+
```
26+
27+
### Links
28+
1. https://stackoverflow.com/questions/67368724/share-same-steps-for-different-github-actions-jobs
29+
2. https://github.community/t/support-for-yaml-anchors/16128/60
30+
3. https://github.com/mithro/actions-includes
31+
4. https://github.com/allejo/gha-workflows

‎.github/build-cpp-filecoin.src.yml

+193
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
name: Fuhon (cpp-Filecoin)
2+
3+
on:
4+
push:
5+
branches: [master]
6+
tags: '**'
7+
pull_request:
8+
branches: [master] ## target branches
9+
10+
jobs:
11+
## GitHub Actions Workflow does not support yaml anchors
12+
## and that is why there is a workaround with make-workflows.sh
13+
## You should `pre-commit install` or use `pre-commit-hook.sh`,
14+
## anyway please read .github/README.md
15+
check_workflow_yaml_coressponds_to_src_yaml:
16+
runs-on: ubuntu-20.04 #ubuntu-latest
17+
#container: ubuntu:latest ## This is required as barrier between AWS-hosted runners and GitHub-hosted runners - they have different set of software, so run in container
18+
name: Check if github workflows were properly made from sources
19+
steps:
20+
- &step_detect_commented_pr
21+
name: REF and SHA of commented PR to ENV
22+
if: github.event.comment
23+
run: >
24+
curl -fsSL ${{github.event.issue.pull_request.url}}
25+
-H "Authorization: token ${{github.token}}" |
26+
jq -r '
27+
"PR_REF="+.head.ref,
28+
"PR_SHA="+.head.sha,
29+
"PR_NUM="+(.number|tostring),
30+
"PR_REPO="+.head.repo.full_name' >>$GITHUB_ENV
31+
- &step_checkout
32+
name: Checkout
33+
uses: actions/checkout@v2
34+
with: &step_checkout_with
35+
ref: ${{env.PR_REF}} ## not empty on issue_comment, else default value GITHUB_REF
36+
repository: ${{env.PR_REPO}} ## not empty on issue_comment, else default value github.repository, required by forks
37+
-
38+
run: sudo snap install yq
39+
-
40+
name: Check if .github/workflows/*.yml correspond to *.src.yml
41+
run: |
42+
set -x
43+
[[ $(./.github/make-workflows.sh -x --worktree) = *"everything is up to date" ]]
44+
45+
## Build docker image named 'filecoin/fuhon-builder' with all stuff to compile fuhon and its dependancies
46+
## The result docker image is pushed with tags :pr-NUMBER, :commit-HASH, :branch-name, :tag-name,
47+
## and conditional tags :edge for development branch, and :latest for git-tags.
48+
## Note: image is push only when DockerHub login-token pair available - not to PRs from forks.
49+
Docker-fuhon-builder:
50+
needs: check_workflow_yaml_coressponds_to_src_yaml
51+
runs-on: ubuntu-20.04 #ubuntu-latest #[ self-hosted, Linux ]
52+
env: &env_dockerhub
53+
DOCKERHUB_ORG: soramitsu ## Cannot use ${{ secrets.DOCKERHUB_ORG }}
54+
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
55+
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
56+
steps:
57+
- &step_system_info
58+
name: System info
59+
run: |
60+
set -x
61+
whoami
62+
id $(whoami)
63+
free || vm_stat | perl -ne '/page size of (\d+)/ and $size=$1;
64+
/Pages\s+([^:]+)[^\d]+(\d+)/ and printf("%-16s % 16.2f Mi\n", "$1:", $2 * $size / 1048576);'
65+
df -h
66+
- &step_build_info
67+
name: Build info
68+
run: |
69+
cat << 'END'
70+
ref:${{github.ref}}
71+
sha:${{github.sha}}
72+
run_number:${{github.run_number}}
73+
event_name:${{github.event_name}}
74+
event.action:${{github.event.action}}
75+
event.issue.number:${{ github.event.issue.number }}
76+
END
77+
- *step_detect_commented_pr
78+
- *step_checkout
79+
- &step_docker_tag
80+
name: Determine dockertag
81+
id: dockertag
82+
env:
83+
dockertag: ${{ hashFiles('docker/**') }}
84+
run: |
85+
echo "::set-output name=dockertag::$dockertag"
86+
echo >>$GITHUB_ENV dockertag=$dockertag
87+
test -n "$DOCKERHUB_ORG" || {
88+
echo ::error::"DOCKERHUB_ORG must contain value"
89+
false
90+
}
91+
- &step_docker_login
92+
name: Login to DockerHub
93+
if: ${{ env.DOCKERHUB_TOKEN != '' && env.DOCKERHUB_USERNAME != '' }}
94+
id: docker_login
95+
uses: docker/login-action@v1
96+
with:
97+
registry: docker.soramitsu.co.jp
98+
username: ${{ secrets.DOCKERHUB_USERNAME }}
99+
password: ${{ secrets.DOCKERHUB_TOKEN }}
100+
- &step_warn_docker_no_push
101+
name: Possible WARNING
102+
if: ${{ steps.docker_login.outcome == 'skipped' }}
103+
run: echo "::warning::DOCKERHUB_TOKEN and DOCKERHUB_USERNAME are empty. Will build but NOT push."
104+
- &step_docker_meta
105+
name: Docker meta
106+
id: meta
107+
uses: docker/metadata-action@v3
108+
with: &step_docker_meta_with
109+
# registry.rocket.chat/rocketchat/rocket.chat:latest
110+
images: docker.soramitsu.co.jp/fuhon/node
111+
tags: |
112+
type=raw,value=${{env.dockertag}}
113+
type=ref,event=branch
114+
type=ref,event=pr
115+
type=ref,event=tag
116+
type=schedule
117+
type=edge,branch=develop
118+
## Docker image will be pushed with tags:
119+
## - hash of file Dockerfile.builder
120+
## - branchname, when branch is pushed
121+
## - pr-NUMBER, when pushed to PR
122+
## - git tag when tag is pushed
123+
## - semver like 1.2.3 and 1.2 when tag vX.X.X is pushed
124+
## - tag 'edge' when branch support/1.2.x is pushed
125+
## - schedule, see the docs
126+
- &step_docker_buildx
127+
name: Set up Docker Buildx
128+
uses: docker/setup-buildx-action@v1
129+
- &step_docker_cache
130+
name: Cache Docker layers
131+
uses: actions/cache@v2
132+
with:
133+
path: /tmp/.buildx-cache
134+
key: ${{ runner.os }}-buildx-${{env.dockertag}}
135+
restore-keys: ${{ runner.os }}-buildx-
136+
- &step_docker_build_and_push
137+
id: build_and_push
138+
name: Build and push
139+
uses: docker/build-push-action@v2
140+
with: &step_docker_build_and_push_with
141+
cache-from: type=local,src=/tmp/.buildx-cache
142+
cache-to: type=local,dest=/tmp/.buildx-cache-new
143+
push: ${{ steps.docker_login.outcome == 'success' }}
144+
tags: ${{ steps.meta.outputs.tags }}
145+
labels: ${{ steps.meta.outputs.labels }}
146+
context: .
147+
- &step_docker_move_cache
148+
# Temp fix
149+
# https://github.com/docker/build-push-action/issues/252
150+
# https://github.com/moby/buildkit/issues/1896
151+
name: Move cache
152+
run: |
153+
rm -rf /tmp/.buildx-cache
154+
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
155+
-
156+
name: Check if dockertaghash exists in remote registry
157+
id: dockertag_already
158+
run: |
159+
echo "::set-output name=container::$DOCKERHUB_ORG/fuhon:$dockertag"
160+
-
161+
name: Possible ERROR, Dockerfile edited but image cannot be pushed
162+
if: ${{ steps.docker_login.outcome != 'success' || steps.build_and_push.outcome != 'success' }}
163+
env:
164+
container: ${{steps.dockertag_already.outputs.container}}
165+
dockertag: ${{env.dockertag}}
166+
run: |
167+
cat <<END
168+
::error::CHANGES TO Dockerfile.builder WERE NOT APPLYED.
169+
170+
It seems container with tag '$dockertag' was not pushed to registry and does not exist remotely.
171+
The most possible reason is GitHub secrets are inaccessable to PRs from public forks.
172+
173+
$(test ${{github.event.pull_request.head.repo.full_name}} != ${{github.event.pull_request.base.repo.full_name}} \
174+
&& echo -n "SECRETS ARE NOT EXPOSED TO FORKS" || echo -n "SECRETS AVAILABLE")
175+
176+
**Consider to open PR from the same organization.**
177+
178+
What we know about this build:
179+
- PR URL is ${{github.event.pull_request.html_url}}
180+
- head repo is '${{github.event.pull_request.head.repo.full_name}}'
181+
- base repo is '${{github.event.pull_request.base.repo.full_name}}'
182+
- See more information in previous step 'Show context'
183+
184+
Please ask @ahmedelkashev on GitHub or in Telegram if you need help.
185+
END
186+
false
187+
outputs:
188+
## WARN secret dropped from output!, output may not contain secret,
189+
## and secret cannot be used in job:container directly, and there is no github non-secret variables...
190+
## if dockertag is already pushed then use it. But let it be empty when tag does not exist remotely.
191+
dockertag: ${{steps.dockertag.outputs.dockertag}}
192+
container: ${{steps.dockertag_already.outputs.container}}
193+
pushed: ${{ steps.docker_login.outcome == 'success' && steps.build_and_push.outcome == 'success' }}

‎.github/disabled_workflows/clang-tidy.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
name: checkout repo
1919
- name: install
2020
run: |
21-
brew install ninja llvm pkg-config
21+
brew install ninja llvm pkg-config coreutils
2222
sudo python3 -m pip install --upgrade pip
2323
sudo python3 -m pip install scikit-build cmake requests gitpython gcovr pyyaml
2424
- name: run checks

‎.github/make-workflows.sh

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
5+
--help(){
6+
cat<<'END'
7+
make-workflows:
8+
This script expands '*.src.yml' from $1 (default: script's directory)
9+
to $2 (default:REPO_ROOT/.github/workflows) with corresponding name '*.yml'
10+
Main goal is to dereference YAML anchors.
11+
Deals only with Git cached/indexed files until --no-git-index passed.
12+
DEBUG: use option -x
13+
NOTE: spaces in filenames are not allowed to keep code simplicity
14+
END
15+
cat<<END
16+
Usage:
17+
make-workflows.sh [--no-git-index] [dirs_from... [dir_to]]
18+
make-workflows.sh [--help]
19+
Options:
20+
--no-git-index|--worktree List files and get contents from working tree
21+
instead of git index
22+
-h, --help show this help
23+
-x, --trace
24+
+x, --no-trace enable/disable bash trace
25+
END
26+
exit
27+
}
28+
29+
files_list(){
30+
git diff --cached --name-only --relative --diff-filter=d -- "$@"
31+
## NOTE: --diff-filter=d to exclude deleted files
32+
}
33+
file_contents(){
34+
git show $(printf ":%s " $@)
35+
}
36+
37+
while [[ $# > 0 ]] ;do
38+
case "$1" in
39+
## List files and get contents from working tree instead of git index
40+
--no-git-index|--worktree)
41+
files_list(){
42+
ls $@
43+
}
44+
file_contents(){
45+
cat $@
46+
}
47+
;;
48+
-x|--trace) set -x ;;
49+
+x|--no-trace) set +x ;;
50+
-h|--help|'-?') --help ;;
51+
-*)
52+
echo >&2 "make-workflows: ERROR: unxpected parameter"
53+
--help >&2
54+
exit 2
55+
;;
56+
## The last non-option argument is dir_to all previous are dirs_from
57+
*)
58+
if [[ "$1" = *' '* ]] ;then
59+
echo >&2 "make-workflows: ERROR: spaces in arguments are not allowed: '$1'"
60+
exit 1
61+
fi
62+
if [[ "$(echo ${dirs_from:-})" = '' ]] ;then
63+
dirs_from=$1
64+
else
65+
dirs_from+=" "${dir_to:-}
66+
dir_to=$1
67+
fi
68+
;;
69+
esac
70+
shift
71+
done
72+
73+
readonly script_dir=$(dirname $(realpath "$0"))
74+
readonly dirs_from=${dirs_from:-${script_dir}}
75+
readonly repo_root=$(git rev-parse --show-toplevel)
76+
dir_to=${dir_to:-$repo_root/.github/workflows}
77+
readonly dir_to=$(realpath $dir_to)
78+
edited_files=
79+
80+
for dir_from in $dirs_from ;do
81+
pushd $dir_from >/dev/null
82+
for f in $(files_list '*.src.yml') ;do
83+
out=$(echo $f | sed 's|.src.yml$|.yml|')
84+
wout=$dir_to/$out
85+
tempout=$(mktemp)
86+
trap "rm -f $tempout" EXIT ## in case of error file will be removed before exit
87+
echo >>$tempout "## DO NOT EDIT"
88+
echo >>$tempout "## Generated from $f with $(basename $0)"
89+
echo >>$tempout ""
90+
## Take cached content from index
91+
file_contents ./$f | yq eval 'explode(.)' - >>$tempout
92+
if ! diff -q $wout $tempout &>/dev/null ;then
93+
mv $tempout $wout
94+
edited_files+="'$(realpath --relative-to=$OLDPWD $wout)' "
95+
else
96+
rm -f $tempout
97+
fi
98+
done
99+
popd >/dev/null
100+
done
101+
102+
if [[ -n "$edited_files" ]]
103+
then echo "make-workflows: these files were edited: $edited_files"
104+
else echo "make-workflows: everything is up to date"
105+
fi

‎.github/pre-commit-hook.sh

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
cd $(git rev-parse --show-toplevel)
4+
./.github/make-workflows.sh
5+
git add .github/workflows

0 commit comments

Comments
 (0)