Skip to content

Commit 39b5c4e

Browse files
jeregrinejeregrine
authored
Merge pull request #5 from phoenixframework/add-ci-matrix
Add Elixir and OTP CI coverage and stabilize distributed tests
2 parents 78f50d0 + c26e9d3 commit 39b5c4e

18 files changed

Lines changed: 1186 additions & 277 deletions

‎.formatter.exs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
# Used by "mix format"
22
[
3-
inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
3+
inputs: [
4+
"{mix,.formatter}.exs",
5+
"{config,lib,test}/**/*.{ex,exs}",
6+
"priv/bench/comparison.exs"
7+
]
48
]

‎.github/workflows/campaigns.yml‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
name: Fault campaigns
2+
3+
on:
4+
schedule:
5+
- cron: "23 3 * * *"
6+
workflow_dispatch:
7+
release:
8+
types: [published]
9+
10+
permissions:
11+
contents: read
12+
13+
concurrency:
14+
group: ${{ github.workflow }}-${{ github.ref }}
15+
cancel-in-progress: false
16+
17+
defaults:
18+
run:
19+
shell: bash
20+
21+
env:
22+
MIX_ENV: test
23+
ERL_FLAGS: "+S 4:4"
24+
GROUP_HISTORY_STEPS: "2000"
25+
GROUP_TEST_DIAGNOSTICS: ${{ github.workspace }}/_build/test-diagnostics
26+
ERL_CRASH_DUMP: ${{ github.workspace }}/_build/test-diagnostics/erl_crash.dump
27+
28+
jobs:
29+
histories-and-faults:
30+
name: Seed ${{ matrix.seed }} / ${{ matrix.schedulers }} peer schedulers
31+
runs-on: ubuntu-24.04
32+
timeout-minutes: 30
33+
strategy:
34+
fail-fast: false
35+
matrix:
36+
seed: [104729, 130363, 155921]
37+
schedulers: ["1", "2", "4"]
38+
env:
39+
GROUP_PEER_SCHEDULERS: ${{ matrix.schedulers }}
40+
CAMPAIGN_SEED: ${{ matrix.seed }}
41+
steps:
42+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
43+
with:
44+
persist-credentials: false
45+
- uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
46+
with:
47+
elixir-version: "> 0"
48+
otp-version: "> 0"
49+
- uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
50+
with:
51+
distribution: temurin
52+
java-version: "21"
53+
- run: mix deps.get
54+
- name: Repeat histories, partitions, churn, restarts and race regressions
55+
run: |
56+
mkdir -p "$GROUP_TEST_DIAGNOSTICS"
57+
mix compile --warnings-as-errors
58+
for iteration in 0 1 2 3 4; do
59+
seed=$((CAMPAIGN_SEED + iteration))
60+
mix test --warnings-as-errors --seed "$seed" 2>&1 |
61+
tee "$GROUP_TEST_DIAGNOSTICS/run-$seed.log"
62+
done
63+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
64+
if: always()
65+
with:
66+
name: campaign-${{ matrix.seed }}-${{ matrix.schedulers }}
67+
path: _build/test-diagnostics
68+
retention-days: 14
69+
70+
qualification:
71+
name: Protocol mutations and live checker qualification
72+
runs-on: ubuntu-24.04
73+
timeout-minutes: 90
74+
env:
75+
GROUP_HISTORY_STEPS: "100"
76+
GROUP_PEER_SCHEDULERS: "2"
77+
steps:
78+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
79+
with:
80+
persist-credentials: false
81+
- uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
82+
with:
83+
elixir-version: "> 0"
84+
otp-version: "> 0"
85+
- uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
86+
with:
87+
distribution: temurin
88+
java-version: "21"
89+
- run: mix deps.get
90+
- name: Run the upstream PR gate before qualification
91+
run: |
92+
mkdir -p "$GROUP_TEST_DIAGNOSTICS"
93+
mix test --warnings-as-errors 2>&1 | tee "$GROUP_TEST_DIAGNOSTICS/baseline.log"
94+
- name: Require protocol mutants and injected checker faults to be detected
95+
run: |
96+
GROUP_JEPSEN_SKIP_CHECKER=1 test/jepsen/qualify.sh 2>&1 |
97+
tee "$GROUP_TEST_DIAGNOSTICS/qualification.log"
98+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
99+
if: always()
100+
with:
101+
name: qualification-reports
102+
path: |
103+
_build/test-diagnostics
104+
tmp/mutation/**/*.log
105+
test/jepsen/.cache/qualification.*
106+
test/jepsen/store
107+
include-hidden-files: true
108+
retention-days: 14
109+
110+
soak:
111+
name: Jepsen ${{ matrix.transport }} / ${{ matrix.scenario }}
112+
needs: qualification
113+
runs-on: ubuntu-24.04
114+
timeout-minutes: 180
115+
strategy:
116+
fail-fast: false
117+
matrix:
118+
transport: [distribution, tcp, chaos]
119+
scenario: [mixed, permanent]
120+
env:
121+
GROUP_JEPSEN_SKIP_CHECKER: "1"
122+
GROUP_JEPSEN_CAMPAIGN_TRANSPORT: ${{ matrix.transport }}
123+
GROUP_JEPSEN_CAMPAIGN_SCENARIO: ${{ matrix.scenario }}
124+
GROUP_JEPSEN_CAMPAIGN_ARTIFACT_DIR: ${{ github.workspace }}/_build/jepsen-campaign
125+
steps:
126+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
127+
with:
128+
persist-credentials: false
129+
- uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
130+
with:
131+
elixir-version: "> 0"
132+
otp-version: "> 0"
133+
- uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
134+
with:
135+
distribution: temurin
136+
java-version: "21"
137+
- run: mix deps.get
138+
# Split the upstream six-profile gate across jobs without reducing its
139+
# default 20 five-minute histories per profile.
140+
- run: test/jepsen/campaign.sh
141+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
142+
if: always()
143+
with:
144+
name: soak-${{ matrix.transport }}-${{ matrix.scenario }}
145+
path: |
146+
_build/jepsen-campaign
147+
test/jepsen/store
148+
retention-days: 14

‎.github/workflows/ci.yml‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
name: CI
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
release:
8+
types: [published]
9+
workflow_dispatch:
10+
11+
permissions:
12+
contents: read
13+
14+
concurrency:
15+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
16+
cancel-in-progress: true
17+
18+
defaults:
19+
run:
20+
shell: bash
21+
22+
env:
23+
MIX_ENV: test
24+
ERL_FLAGS: "+S 4:4"
25+
GROUP_PEER_SCHEDULERS: "2"
26+
GROUP_TEST_DIAGNOSTICS: ${{ github.workspace }}/_build/test-diagnostics
27+
ERL_CRASH_DUMP: ${{ github.workspace }}/_build/test-diagnostics/erl_crash.dump
28+
29+
jobs:
30+
local:
31+
name: Local checks
32+
runs-on: ubuntu-24.04
33+
timeout-minutes: 5
34+
steps:
35+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
36+
with:
37+
persist-credentials: false
38+
- uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
39+
with:
40+
elixir-version: "> 0"
41+
otp-version: "> 0"
42+
- uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
43+
with:
44+
distribution: temurin
45+
java-version: "21"
46+
- run: mix deps.get
47+
- name: Formatting, compiler checks, local examples and histories
48+
run: |
49+
mkdir -p "$GROUP_TEST_DIAGNOSTICS"
50+
mix format --check-formatted 2>&1 | tee "$GROUP_TEST_DIAGNOSTICS/format.log"
51+
mix compile --warnings-as-errors 2>&1 | tee "$GROUP_TEST_DIAGNOSTICS/compile.log"
52+
mix test --only local --warnings-as-errors 2>&1 | tee "$GROUP_TEST_DIAGNOSTICS/local.log"
53+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
54+
if: failure()
55+
with:
56+
name: local-failure
57+
path: _build/test-diagnostics
58+
retention-days: 14
59+
60+
test:
61+
name: ${{ matrix.latest && 'Latest stable Elixir / OTP' || format('Elixir {0} / OTP {1}', matrix.elixir, matrix.otp) }}
62+
needs: local
63+
runs-on: ubuntu-24.04
64+
timeout-minutes: 15
65+
strategy:
66+
fail-fast: false
67+
matrix:
68+
include:
69+
- elixir: "1.19"
70+
otp: "26"
71+
- elixir: "1.19"
72+
otp: "27"
73+
- elixir: "1.19"
74+
otp: "28"
75+
- elixir: "1.20"
76+
otp: "27"
77+
- elixir: "1.20"
78+
otp: "28"
79+
# Unlike "latest", these ranges exclude prereleases.
80+
- elixir: "> 0"
81+
otp: "> 0"
82+
latest: true
83+
steps:
84+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
85+
with:
86+
persist-credentials: false
87+
88+
- uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
89+
with:
90+
elixir-version: ${{ matrix.elixir }}
91+
otp-version: ${{ matrix.otp }}
92+
93+
- uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1
94+
with:
95+
distribution: temurin
96+
java-version: "21"
97+
98+
- name: Install dependencies
99+
run: mix deps.get
100+
101+
- name: Compile with warnings as errors
102+
run: |
103+
mkdir -p "$GROUP_TEST_DIAGNOSTICS"
104+
mix compile --warnings-as-errors 2>&1 | tee "$GROUP_TEST_DIAGNOSTICS/compile.log"
105+
106+
- name: Run local and distributed tests
107+
run: mix test --warnings-as-errors 2>&1 | tee "$GROUP_TEST_DIAGNOSTICS/test.log"
108+
109+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
110+
if: failure()
111+
with:
112+
name: runtime-failure-${{ strategy.job-index }}
113+
path: _build/test-diagnostics
114+
retention-days: 14

‎.github/workflows/performance.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
name: Performance comparison
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
baseline:
7+
description: "Baseline ref (use a previous release tag for release review)"
8+
type: string
9+
default: main
10+
required: true
11+
release:
12+
types: [published]
13+
14+
permissions:
15+
contents: read
16+
17+
defaults:
18+
run:
19+
shell: bash
20+
21+
jobs:
22+
compare:
23+
runs-on: ubuntu-24.04
24+
timeout-minutes: 60
25+
env:
26+
ERL_FLAGS: "+S 4:4"
27+
steps:
28+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
29+
with:
30+
path: candidate
31+
fetch-depth: 0
32+
persist-credentials: false
33+
- name: Select baseline
34+
id: baseline
35+
env:
36+
BASELINE: ${{ inputs.baseline }}
37+
run: |
38+
if [ -z "$BASELINE" ]; then
39+
BASELINE=$(git -C candidate describe --tags --abbrev=0 "${GITHUB_SHA}^")
40+
fi
41+
printf 'ref=%s\n' "$BASELINE" >> "$GITHUB_OUTPUT"
42+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
43+
with:
44+
path: baseline
45+
ref: ${{ steps.baseline.outputs.ref }}
46+
persist-credentials: false
47+
- uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
48+
with:
49+
elixir-version: "> 0"
50+
otp-version: "> 0"
51+
- name: Record environment and revisions
52+
run: |
53+
mkdir -p reports
54+
{
55+
elixir --version
56+
uname -a
57+
lscpu
58+
git -C baseline rev-parse HEAD
59+
git -C candidate rev-parse HEAD
60+
} | tee reports/environment.log
61+
- name: Benchee baseline and candidate on the same runner
62+
run: |
63+
GROUP_BENCH_ROOT="$PWD/baseline" \
64+
GROUP_BENCH_OUTPUT="$PWD/reports/baseline.benchee" \
65+
GROUP_BENCH_TAG=baseline \
66+
elixir candidate/priv/bench/comparison.exs 2>&1 | tee reports/benchee-baseline.log
67+
GROUP_BENCH_ROOT="$PWD/candidate" \
68+
GROUP_BENCH_OUTPUT="$PWD/reports/candidate.benchee" \
69+
GROUP_BENCH_BASELINE="$PWD/reports/baseline.benchee" \
70+
elixir candidate/priv/bench/comparison.exs 2>&1 | tee reports/benchee-candidate.log
71+
- name: Distributed load and recovery, baseline then candidate
72+
run: |
73+
epmd -daemon
74+
for revision in baseline candidate; do
75+
timeout 20m bash "$revision/priv/bench/run_distributed.sh" 2>&1 |
76+
tee "reports/distributed-$revision.log"
77+
done
78+
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
79+
if: always()
80+
with:
81+
name: performance-comparison
82+
path: reports
83+
retention-days: 30

‎README.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -619,6 +619,28 @@ See [`test/README.md`](test/README.md) for the every-PR gate, shrinkable
619619
StreamData lifecycle-model tests, bounded TLA+ models, and the nightly
620620
three-node Jepsen transport/lifecycle campaign.
621621

622+
GitHub Actions runs fast local checks first, then the full suite on pull requests,
623+
pushes to `main`, and releases using Elixir 1.19 / OTP 26–28, Elixir 1.20 / OTP 27–28, and the latest
624+
stable Elixir / OTP pair. Version ranges pick up new patch releases automatically;
625+
the latest-stable job also picks up new minor and major releases, excluding
626+
prereleases. Every job treats compilation and test warnings as errors. Nightly
627+
campaigns vary seeds and peer scheduler counts, run larger generated histories,
628+
and check targeted mutations. Performance comparisons run separately.
629+
See [`test/README.md`](test/README.md#ci-lanes) for lane budgets and release review.
630+
631+
To run the same checks locally:
632+
633+
```bash
634+
export MIX_ENV=test
635+
export ERL_FLAGS="+S 4:4"
636+
export GROUP_PEER_SCHEDULERS=2
637+
mix deps.get
638+
mix format --check-formatted
639+
mix compile --warnings-as-errors
640+
mix test --only local --warnings-as-errors # no distribution startup
641+
mix test --warnings-as-errors
642+
```
643+
622644
## Benchmarks
623645

624646
```bash

0 commit comments

Comments
 (0)