Skip to content

Commit 0c1ff30

Browse files
bmastbergenbmastbergen
committed
Try parrallel testing
1 parent 1e086a0 commit 0c1ff30

File tree

2 files changed

+344
-2
lines changed

2 files changed

+344
-2
lines changed

‎.github/workflows/clk-rebase-parallel.yml‎

Lines changed: 344 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,344 @@
1+
name: CLK Rebase (Parallel Tests)
2+
on:
3+
workflow_dispatch:
4+
push:
5+
branches:
6+
- stable_6.*.y
7+
- clk-rebase-ga
8+
9+
jobs:
10+
rebase-and-build:
11+
runs-on: kernel-build
12+
container:
13+
image: rockylinux:9
14+
options: --cpus 8 --privileged
15+
env:
16+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
17+
# For push events, use the branch that triggered the workflow
18+
# For manual dispatch, default to stable_6.12.y
19+
#STABLE_TRACKING_BRANCH: ${{ github.ref_name || 'stable_6.12.y' }}
20+
STABLE_TRACKING_BRANCH: ${{ 'stable_6.12.y' }}
21+
outputs:
22+
stable-version: ${{ steps.save-versions.outputs.stable-version }}
23+
clk-branch: ${{ steps.save-versions.outputs.clk-branch }}
24+
clk-next-branch: ${{ steps.save-versions.outputs.clk-next-branch }}
25+
tmp-clk-next-branch: ${{ steps.save-versions.outputs.tmp-clk-next-branch }}
26+
steps:
27+
- name: Set version variables
28+
run: |
29+
# Extract version from STABLE_TRACKING_BRANCH (e.g., stable_6.12.y -> 6.12.y)
30+
STABLE_BASE_VERSION=$(echo "$STABLE_TRACKING_BRANCH" | sed 's/^stable_//')
31+
echo "STABLE_BASE_VERSION=$STABLE_BASE_VERSION" >> $GITHUB_ENV
32+
33+
# Construct branch names from the base version
34+
echo "CLK_BRANCH=ciq-$STABLE_BASE_VERSION" >> $GITHUB_ENV
35+
echo "CLK_NEXT_BRANCH=ciq-${STABLE_BASE_VERSION}-next" >> $GITHUB_ENV
36+
echo "TMP_CLK_NEXT_BRANCH={automation_tmp}_ciq-${STABLE_BASE_VERSION}-next" >> $GITHUB_ENV
37+
38+
- name: Install system dependencies
39+
run: |
40+
dnf install -y epel-release
41+
dnf groupinstall -y 'Development Tools'
42+
dnf install -y --enablerepo=crb bc dwarves elfutils-libelf-devel grubby grub2-tools kernel-devel openssl-devel
43+
44+
- name: Checkout kernel-src-tree
45+
uses: actions/checkout@v4
46+
with:
47+
repository: ctrliq/kernel-src-tree
48+
token: ${{ secrets.GITHUB_TOKEN }}
49+
path: kernel-src-tree
50+
fetch-depth: 0
51+
52+
- name: Checkout kernel-src-tree-tools
53+
uses: actions/checkout@v4
54+
with:
55+
repository: ctrliq/kernel-src-tree-tools
56+
token: ${{ secrets.GITHUB_TOKEN }}
57+
path: kernel-src-tree-tools
58+
59+
- name: Configure git
60+
run: |
61+
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
62+
git config --global user.name "github-actions[bot]"
63+
64+
- name: Perform rebase
65+
run: |
66+
cd kernel-src-tree
67+
68+
git checkout $STABLE_TRACKING_BRANCH
69+
git checkout $CLK_BRANCH
70+
71+
../kernel-src-tree-tools/lt_rebase.sh
72+
73+
- name: Build kernel
74+
run: |
75+
cd kernel-src-tree
76+
77+
# need some config tweaks for vng
78+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_VSOCKETS
79+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_VIRTIO_VSOCKETS
80+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_VIRTIO_VSOCKETS_COMMON
81+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_NET_9P
82+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_NET_9P_FD
83+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_NET_9P_VIRTIO
84+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_FAILOVER
85+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_BLK_SD_DEV
86+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_SCSI_VIRTIO
87+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_VIRTIO_NET
88+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_NET_FAILOVER
89+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_VIRTIO_CONSOLE
90+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_I6300ESB_WDT
91+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_VIRTIO_BALLOON
92+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_VIRTIO_MMIO
93+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_FUSE_FS
94+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_VIRTIO_FS
95+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_OVERLAY_FS
96+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_NETFS_SUPPORT
97+
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_9P_FS
98+
99+
../kernel-src-tree-tools/kernel_build.sh --skip-kabi --no-reboot | tee ../build.log
100+
101+
- name: Save version information
102+
id: save-versions
103+
run: |
104+
cd kernel-src-tree
105+
106+
# Extract the stable version we rebased onto
107+
STABLE_VERSION=$(git log -1 --format=%s $STABLE_TRACKING_BRANCH | grep -oP 'Linux \K[0-9]+\.[0-9]+\.[0-9]+' || echo "unknown")
108+
echo "stable-version=$STABLE_VERSION" >> $GITHUB_OUTPUT
109+
echo "clk-branch=$CLK_BRANCH" >> $GITHUB_OUTPUT
110+
echo "clk-next-branch=$CLK_NEXT_BRANCH" >> $GITHUB_OUTPUT
111+
echo "tmp-clk-next-branch=$TMP_CLK_NEXT_BRANCH" >> $GITHUB_OUTPUT
112+
113+
- name: Upload kernel artifacts
114+
uses: actions/upload-artifact@v4
115+
with:
116+
name: kernel-build
117+
path: kernel-src-tree/
118+
retention-days: 1
119+
120+
- name: Upload build log
121+
if: always()
122+
uses: actions/upload-artifact@v4
123+
with:
124+
name: build-log
125+
path: build.log
126+
if-no-files-found: warn
127+
128+
discover-tests:
129+
needs: rebase-and-build
130+
runs-on: kernel-build
131+
container:
132+
image: rockylinux:9
133+
outputs:
134+
matrix: ${{ steps.create-matrix.outputs.matrix }}
135+
steps:
136+
- name: Download kernel build
137+
uses: actions/download-artifact@v4
138+
with:
139+
name: kernel-build
140+
path: kernel-src-tree
141+
142+
- name: Discover available test targets
143+
id: create-matrix
144+
run: |
145+
cd kernel-src-tree/tools/testing/selftests
146+
147+
# Skip list - tests we don't want to run
148+
SKIP_LIST="lkdtm"
149+
150+
# Extract TARGETS from Makefile - use awk to get just the target names
151+
AVAILABLE_TARGETS=$(grep -E '^TARGETS \+?=' Makefile | awk '{print $3}')
152+
153+
# Filter out skip list and create JSON array
154+
MATRIX_JSON="["
155+
FIRST=true
156+
for target in $AVAILABLE_TARGETS; do
157+
skip=false
158+
for skip_item in $SKIP_LIST; do
159+
if [ "$target" = "$skip_item" ]; then
160+
skip=true
161+
break
162+
fi
163+
done
164+
if [ "$skip" = false ]; then
165+
if [ "$FIRST" = false ]; then
166+
MATRIX_JSON="$MATRIX_JSON,"
167+
fi
168+
MATRIX_JSON="$MATRIX_JSON{\"name\":\"$target\",\"target\":\"$target\"}"
169+
FIRST=false
170+
fi
171+
done
172+
MATRIX_JSON="$MATRIX_JSON]"
173+
174+
echo "matrix={\"test\":$MATRIX_JSON}" >> $GITHUB_OUTPUT
175+
echo "Generated test matrix for $(echo $MATRIX_JSON | grep -o '"name"' | wc -l) tests"
176+
177+
test:
178+
needs: [rebase-and-build, discover-tests]
179+
runs-on: kernel-build
180+
container:
181+
image: rockylinux:9
182+
options: --cpus 8 --privileged
183+
strategy:
184+
fail-fast: false
185+
matrix: ${{ fromJson(needs.discover-tests.outputs.matrix) }}
186+
steps:
187+
- name: Install dependencies
188+
run: |
189+
dnf install -y epel-release
190+
dnf groupinstall -y 'Development Tools'
191+
dnf install -y --enablerepo=crb,devel --allowerasing VirtualGL alsa-lib-devel bc clang conntrack-tools curl dropwatch dwarves e2fsprogs elfutils-libelf-devel ethtool fuse fuse-devel gcc-toolset-13-libasan-devel glibc glibc-static iperf3 iproute iptables ipvsadm jq kernel-devel kernel-selftests-internal kernel-tools kmod libasan libcap-devel libcap-ng-devel libmnl-devel libreswan libubsan llvm ncurses-devel net-tools netsniff-ng nftables nmap-ncat numactl-devel openssl-devel packetdrill perf popt-devel python3-pip python3-pyyaml python3-jsonschema qemu-kvm rsync socat sudo tcpdump teamd traceroute virtme-ng wget
192+
dnf install -y --setopt=tsflags=nocaps iputils
193+
pip3 install --user netaddr packaging pyftpdlib pyparsing pytest scapy tftpy wheel
194+
195+
# so root can call sudo without complaint
196+
echo "root ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers
197+
198+
- name: Download kernel artifacts
199+
uses: actions/download-artifact@v4
200+
with:
201+
name: kernel-build
202+
path: kernel-src-tree
203+
204+
- name: Run selftests for ${{ matrix.test.name }}
205+
run: |
206+
cd kernel-src-tree
207+
208+
# Test if vng works at all - just create a dummy log file
209+
vng --qemu /usr/libexec/qemu-kvm --force-initramfs --disable-microvm --rw --network user --verbose --memory 16G -- sh -c "mkdir -p ../kselftest-logs ; echo 'ok test-${{ matrix.test.target }}' >> ../kselftest-logs/selftest-\$(uname -r)-${{ matrix.test.target }}.log"
210+
211+
- name: Upload test logs for ${{ matrix.test.name }}
212+
if: always()
213+
uses: actions/upload-artifact@v4
214+
with:
215+
name: kselftest-logs-${{ matrix.test.name }}
216+
path: kselftest-logs/selftest*
217+
if-no-files-found: warn
218+
219+
aggregate-results:
220+
needs: [rebase-and-build, test]
221+
if: always()
222+
runs-on: kernel-build
223+
container:
224+
image: rockylinux:9
225+
env:
226+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
227+
steps:
228+
- name: Install dependencies
229+
run: |
230+
dnf install -y git 'dnf-command(config-manager)'
231+
dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
232+
dnf install -y gh
233+
234+
- name: Download all test logs
235+
uses: actions/download-artifact@v4
236+
with:
237+
pattern: kselftest-logs-*
238+
path: kselftest-logs/
239+
merge-multiple: true
240+
241+
- name: Download build log
242+
uses: actions/download-artifact@v4
243+
with:
244+
name: build-log
245+
path: .
246+
247+
- name: Download kernel build
248+
uses: actions/download-artifact@v4
249+
with:
250+
name: kernel-build
251+
path: kernel-src-tree
252+
253+
- name: Aggregate test results
254+
run: |
255+
echo "Aggregating test results..."
256+
OK_TESTS=$(grep -a ^ok kselftest-logs/selftest* 2>/dev/null | wc -l || echo "0")
257+
FAIL_TESTS=$(grep -a "^not ok" kselftest-logs/selftest* 2>/dev/null | wc -l || echo "0")
258+
259+
echo "Tests passed: $OK_TESTS"
260+
echo "Tests failed: $FAIL_TESTS"
261+
262+
echo "$OK_TESTS" > ok_tests.txt
263+
echo "$FAIL_TESTS" > fail_tests.txt
264+
265+
- name: Extract config changes and push branches
266+
run: |
267+
cd kernel-src-tree
268+
269+
STABLE_VERSION="${{ needs.rebase-and-build.outputs.stable-version }}"
270+
CLK_NEXT_BRANCH="${{ needs.rebase-and-build.outputs.clk-next-branch }}"
271+
TMP_CLK_NEXT_BRANCH="${{ needs.rebase-and-build.outputs.tmp-clk-next-branch }}"
272+
273+
# Check for config changes
274+
STABLE_TRACKING_BRANCH="stable_6.12.y"
275+
git checkout $STABLE_TRACKING_BRANCH
276+
git checkout $CLK_NEXT_BRANCH || git checkout -b $CLK_NEXT_BRANCH
277+
278+
HEAD_COMMIT_MSG=$(git log -1 --format=%s HEAD)
279+
if echo "$HEAD_COMMIT_MSG" | grep -qF "[CIQ] v${STABLE_VERSION} - rebased configs"; then
280+
echo "Config change commit detected for v${STABLE_VERSION}"
281+
git show HEAD --stat > ../config_changes.txt
282+
else
283+
echo "No config change commit detected for v${STABLE_VERSION}"
284+
echo "None" > ../config_changes.txt
285+
fi
286+
287+
# Push branches
288+
git push origin $CLK_NEXT_BRANCH
289+
git push origin $TMP_CLK_NEXT_BRANCH
290+
291+
- name: Create Pull Request
292+
if: success()
293+
run: |
294+
cd kernel-src-tree
295+
296+
STABLE_VERSION="${{ needs.rebase-and-build.outputs.stable-version }}"
297+
CLK_NEXT_BRANCH="${{ needs.rebase-and-build.outputs.clk-next-branch }}"
298+
TMP_CLK_NEXT_BRANCH="${{ needs.rebase-and-build.outputs.tmp-clk-next-branch }}"
299+
300+
OK_TESTS=$(cat ../ok_tests.txt)
301+
FAIL_TESTS=$(cat ../fail_tests.txt)
302+
CONFIG_CHANGES=$(cat ../config_changes.txt)
303+
304+
# Extract abbreviated build log
305+
BUILD_LOG_SUMMARY=$(egrep -B 5 -A 5 "\[TIMER\]|^Starting Build" ../build.log)
306+
307+
# Get artifact URLs
308+
RUN_ID="${{ github.run_id }}"
309+
REPO="${{ github.repository }}"
310+
SELFTEST_ARTIFACT_URL="https://github.com/${REPO}/actions/runs/${RUN_ID}"
311+
BUILD_LOG_ARTIFACT_URL="https://github.com/${REPO}/actions/runs/${RUN_ID}"
312+
313+
# Create PR body
314+
cat > /tmp/pr_body.md <<EOF
315+
## Automated Rebase to v${STABLE_VERSION}
316+
317+
### Config Changes
318+
\`\`\`
319+
${CONFIG_CHANGES}
320+
\`\`\`
321+
322+
### Build Log
323+
\`\`\`
324+
${BUILD_LOG_SUMMARY}
325+
\`\`\`
326+
327+
### Testing (Parallelized)
328+
- Tests passed: **${OK_TESTS}**
329+
- Tests failed: **${FAIL_TESTS}**
330+
331+
Tests were run in parallel across 7 test groups for faster execution.
332+
333+
### Artifacts
334+
- [Build Log](${BUILD_LOG_ARTIFACT_URL})
335+
- [Selftest Logs](${SELFTEST_ARTIFACT_URL})
336+
337+
EOF
338+
339+
# Create the PR
340+
gh pr create \
341+
--title "TESTING TESTING TESTING [CIQ 6.12] Rebase to v${STABLE_VERSION}" \
342+
--body-file /tmp/pr_body.md \
343+
--base ${CLK_NEXT_BRANCH} \
344+
--head ${TMP_CLK_NEXT_BRANCH}

‎.github/workflows/clk-rebase.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ on:
44
push:
55
branches:
66
- stable_6.*.y
7-
- clk-rebase-ga
87

98
jobs:
109
clk-rebase:
@@ -86,7 +85,6 @@ jobs:
8685
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_VSOCKETS
8786
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_VIRTIO_VSOCKETS
8887
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_VIRTIO_VSOCKETS_COMMON
89-
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_VSOCKETS
9088
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_NET_9P
9189
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_NET_9P_FD
9290
./scripts/config --file ./ciq/configs/kernel-x86_64.config --enable CONFIG_NET_9P_VIRTIO

0 commit comments

Comments
 (0)