Skip to content

Commit a649a36

Browse files
mkannwischermkannwischer
committed
Add EC2 benchmarking
This commit adds benchmarking on - Graviton 2 (t4g.small) - Graviton 3 (c7g.medium) - Graviton 4 (c8g.medium) - AMD EPYC 4th gen (c7a.medium) - Intel Xeon 4th gen (c7i.metal-24xl) - AMD EPYC 3rd gen (c6a.large) - Intel Xeon 3rd gen (c6i.large) With that we have all the platforms that are covered in mlkem-native. Signed-off-by: Matthias J. Kannwischer <[email protected]>
1 parent 5655c2e commit a649a36

File tree

4 files changed

+356
-2
lines changed

4 files changed

+356
-2
lines changed

‎.github/workflows/bench.yml

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,3 +92,67 @@ jobs:
9292
nix-shell: ${{ matrix.target.cross_prefix != '' && 'ci-cross' || 'ci-bench' }}
9393
cross_prefix: ${{ matrix.target.cross_prefix }}
9494
opt: false
95+
ec2_all:
96+
name: ${{ matrix.target.name }} ${{ matrix.opt.name }}
97+
permissions:
98+
contents: write
99+
pull-requests: write
100+
id-token: write
101+
strategy:
102+
fail-fast: false
103+
matrix:
104+
target:
105+
- name: Graviton2
106+
ec2_instance_type: t4g.small
107+
ec2_ami: ubuntu-latest (aarch64)
108+
archflags: -mcpu=cortex-a76 -march=armv8.2-a
109+
cflags: "-flto"
110+
perf: PERF
111+
- name: Graviton3
112+
ec2_instance_type: c7g.medium
113+
ec2_ami: ubuntu-latest (aarch64)
114+
archflags: -march=armv8.4-a+sha3
115+
cflags: "-flto"
116+
perf: PERF
117+
- name: Graviton4
118+
ec2_instance_type: c8g.medium
119+
ec2_ami: ubuntu-latest (aarch64)
120+
archflags: -march=armv9-a+sha3
121+
cflags: "-flto"
122+
perf: PERF
123+
- name: AMD EPYC 4th gen (c7a)
124+
ec2_instance_type: c7a.medium
125+
ec2_ami: ubuntu-latest (x86_64)
126+
archflags: -mavx2 -mbmi2 -mpopcnt -maes -march=znver4
127+
cflags: "-flto"
128+
perf: PMU
129+
- name: Intel Xeon 4th gen (c7i)
130+
ec2_instance_type: c7i.metal-24xl
131+
ec2_ami: ubuntu-latest (x86_64)
132+
archflags: -mavx2 -mbmi2 -mpopcnt -maes -march=sapphirerapids
133+
cflags: "-flto"
134+
perf: PMU
135+
- name: AMD EPYC 3rd gen (c6a)
136+
ec2_instance_type: c6a.large
137+
ec2_ami: ubuntu-latest (x86_64)
138+
archflags: -mavx2 -mbmi2 -mpopcnt -maes -march=znver3
139+
cflags: "-flto"
140+
perf: PMU
141+
- name: Intel Xeon 3rd gen (c6i)
142+
ec2_instance_type: c6i.large
143+
ec2_ami: ubuntu-latest (x86_64)
144+
archflags: -mavx2 -mbmi2 -mpopcnt -maes -march=icelake-server
145+
cflags: "-flto"
146+
perf: PMU
147+
uses: ./.github/workflows/bench_ec2_reusable.yml
148+
if: github.repository_owner == 'pq-code-package' && (github.event.label.name == 'benchmark' || github.ref == 'refs/heads/main')
149+
with:
150+
ec2_instance_type: ${{ matrix.target.ec2_instance_type }}
151+
ec2_ami: ${{ matrix.target.ec2_ami }}
152+
archflags: ${{ matrix.target.archflags }}
153+
cflags: ${{ matrix.target.cflags }}
154+
opt: "all"
155+
store_results: ${{ github.repository_owner == 'pq-code-package' && github.ref == 'refs/heads/main' }} # Only store optimized results
156+
name: ${{ matrix.target.name }}
157+
perf: ${{ matrix.target.perf }}
158+
secrets: inherit

‎.github/workflows/bench_ec2_any.yml

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
3+
name: bench-ec2-any
4+
permissions:
5+
contents: read
6+
on:
7+
workflow_dispatch:
8+
inputs:
9+
name:
10+
description: Alternative name of instance
11+
default: Graviton2
12+
ec2_instance_type:
13+
description: Type if EC2 instance to benchmark on
14+
default: t4g.small
15+
ec2_ami:
16+
description: AMI ID
17+
type: choice
18+
options:
19+
- ubuntu-latest (x86_64)
20+
- ubuntu-latest (aarch64)
21+
- ubuntu-latest (custom AMI)
22+
default: ubuntu-latest (aarch64)
23+
ec2_ami_id:
24+
description: AMI ID
25+
required: false
26+
default: ami-0c4e709339fa8521a
27+
cflags:
28+
description: Custom CFLAGS for compilation
29+
default:
30+
archflags:
31+
description: Custom ARCH flags for compilation
32+
default: ''
33+
opt:
34+
description: Benchmark optimized, non-optimized, or both
35+
type: choice
36+
options:
37+
- all
38+
- opt
39+
- no_opt
40+
bench_extra_args:
41+
description: Additional command line to be appended to `tests bench` script
42+
default: ''
43+
compiler:
44+
description: Compiler to use. When unset, default nix shell is used.
45+
default: ''
46+
additional_packages:
47+
description: Additional packages to install when custom compiler is used.
48+
default: ''
49+
jobs:
50+
bench-ec2-any:
51+
name: Ad-hoc benchmark on $${{ inputs.ec2_instance_type }}
52+
permissions:
53+
contents: 'read'
54+
id-token: 'write'
55+
uses: ./.github/workflows/bench_ec2_reusable.yml
56+
with:
57+
ec2_instance_type: ${{ inputs.ec2_instance_type }}
58+
ec2_ami: ${{ inputs.ec2_ami }}
59+
ec2_ami_id: ${{ inputs.ec2_ami_id }}
60+
cflags: ${{ inputs.cflags }}
61+
archflags: ${{ inputs.archflags }}
62+
opt: ${{ inputs.opt }}
63+
name: ${{ inputs.name }}
64+
store_results: false
65+
bench_extra_args: ${{ inputs.bench_extra_args }}
66+
compiler: ${{ inputs.compiler }}
67+
additional_packages: ${{ inputs.additional_packages }}
68+
secrets: inherit

‎.github/workflows/bench_ec2_reusable.yml

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
3+
name: bench-ec2-reusable
4+
on:
5+
workflow_call:
6+
inputs:
7+
name:
8+
type: string
9+
description: Alternative name of instance
10+
default: Graviton2
11+
ec2_instance_type:
12+
type: string
13+
description: Type if EC2 instance to benchmark on
14+
default: t4g.small
15+
ec2_ami:
16+
type: string
17+
description: Textual description of AMI
18+
default: ubuntu-latest (aarch64)
19+
ec2_ami_id:
20+
type: string
21+
description: AMI ID
22+
default: ami-0c4e709339fa8521a
23+
cflags:
24+
type: string
25+
description: Custom CFLAGS for compilation
26+
default: ""
27+
archflags:
28+
type: string
29+
description: Custom ARCH flags for compilation
30+
default: -mcpu=neoverse-n1 -march=armv8.2-a
31+
opt:
32+
type: string
33+
description: Runs with optimized code if enabled (opt, no_opt, all)
34+
default: "opt"
35+
perf:
36+
type: string
37+
description: Method by which clock cycles should be measured (PMU | PERF)
38+
default: PERF
39+
store_results:
40+
type: boolean
41+
description: Indicates if results should be pushed to github pages
42+
default: false
43+
verbose:
44+
description: Determine for the log verbosity
45+
type: boolean
46+
default: false
47+
bench_extra_args:
48+
type: string
49+
description: Additional command line to be appended to `bench` script
50+
default: ''
51+
compiler:
52+
type: string
53+
description: Compiler to use. When unset, default nix shell is used.
54+
default: ''
55+
additional_packages:
56+
type: string
57+
description: Additional packages to install when custom compiler is used.
58+
default: ''
59+
aws_region:
60+
type: string
61+
default: "us-east-1"
62+
alert_threshold:
63+
type: string
64+
description: "Set alert threshold in percentage for benchmark result"
65+
default: "103%"
66+
env:
67+
AWS_ROLE: arn:aws:iam::904233116199:role/mldsa-native-ci
68+
AMI_UBUNTU_LATEST_X86_64: ami-084568db4383264d4
69+
AMI_UBUNTU_LATEST_AARCH64: ami-0c4e709339fa8521a
70+
71+
permissions:
72+
contents: read
73+
pull-requests: read
74+
75+
jobs:
76+
start-ec2-runner:
77+
name: Start ${{ inputs.name }} (${{ inputs.ec2_instance_type }})
78+
permissions:
79+
contents: 'read'
80+
id-token: 'write'
81+
runs-on: ubuntu-latest
82+
if: ${{ always() }} # The point is to make this step non-cancellable,
83+
# avoiding race conditions where an instance is started,
84+
# but isn't yet done registering as a runner and reporting back.
85+
outputs:
86+
label: ${{ steps.start-ec2-runner.outputs.label }}
87+
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
88+
steps:
89+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
90+
- name: Determine AMI ID
91+
id: det_ami_id
92+
run: |
93+
if [[ "${{ inputs.ec2_ami }}" == "ubuntu-latest (x86_64)" ]]; then
94+
AMI_ID=${{ env.AMI_UBUNTU_LATEST_X86_64 }}
95+
elif [[ "${{ inputs.ec2_ami }}" == "ubuntu-latest (aarch64)" ]]; then
96+
AMI_ID=${{ env.AMI_UBUNTU_LATEST_AARCH64 }}
97+
elif [[ "${{ inputs.ec2_ami }}" == "ubuntu-latest (custom AMI)" ]]; then
98+
AMI_ID=${{ inputs.ec2_ami_id }}
99+
fi
100+
echo "Using AMI ID: $AMI_ID"
101+
echo "AMI_ID=$AMI_ID" >> $GITHUB_OUTPUT
102+
- name: Configure AWS credentials
103+
uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
104+
with:
105+
role-to-assume: ${{ env.AWS_ROLE }}
106+
aws-region: ${{ inputs.aws_region }}
107+
- name: Start EC2 runner
108+
id: start-ec2-runner
109+
uses: machulav/ec2-github-runner@a8c20fc0876503410b2b966c124abc2311984ce2 # v2.3.9
110+
with:
111+
mode: start
112+
github-token: ${{ secrets.AWS_GITHUB_TOKEN }}
113+
ec2-image-id: ${{ steps.det_ami_id.outputs.AMI_ID }}
114+
ec2-instance-type: ${{ inputs.ec2_instance_type }}
115+
subnet-id: subnet-094d73eb42eb6bf5b
116+
security-group-id: sg-0282706dbc92a1579
117+
bench_nix:
118+
name: Bench (nix)
119+
permissions:
120+
contents: write
121+
pull-requests: write
122+
runs-on: ${{ needs.start-ec2-runner.outputs.label }}
123+
needs: start-ec2-runner # required to start the main job when the runner is ready
124+
if: ${{ inputs.compiler == '' }}
125+
steps:
126+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
127+
- uses: ./.github/actions/bench
128+
if: ${{ inputs.opt == 'all' || inputs.opt == 'opt' }}
129+
with:
130+
nix-verbose: ${{ inputs.verbose }}
131+
name: ${{ inputs.name }}
132+
cflags: ${{ inputs.cflags }}
133+
archflags: ${{ inputs.archflags }}
134+
opt: true
135+
perf: ${{ inputs.perf }}
136+
store_results: ${{ inputs.store_results }}
137+
bench_extra_args: ${{ inputs.bench_extra_args }}
138+
gh_token: ${{ secrets.AWS_GITHUB_TOKEN }}
139+
- uses: ./.github/actions/bench
140+
if: ${{ inputs.opt == 'all' || inputs.opt == 'no_opt' }}
141+
with:
142+
nix-verbose: ${{ inputs.verbose }}
143+
name: ${{ inputs.name }} (no-opt)
144+
cflags: ${{ inputs.cflags }}
145+
archflags: ${{ inputs.archflags }}
146+
opt: false
147+
perf: ${{ inputs.perf }}
148+
store_results: ${{ inputs.store_results }}
149+
bench_extra_args: ${{ inputs.bench_extra_args }}
150+
gh_token: ${{ secrets.AWS_GITHUB_TOKEN }}
151+
alert_threshold: ${{ inputs.alert_threshold }}
152+
bench_custom:
153+
name: Bench (custom compiler)
154+
permissions:
155+
contents: write
156+
pull-requests: write
157+
runs-on: ${{ needs.start-ec2-runner.outputs.label }}
158+
needs: start-ec2-runner # required to start the main job when the runner is ready
159+
if: ${{ inputs.compiler != '' }}
160+
steps:
161+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
162+
- uses: ./.github/actions/setup-apt
163+
with:
164+
packages: ${{ inputs.additional_packages }}
165+
- name: Set compiler
166+
run: |
167+
echo "CC=${{ inputs.compiler }}" >> "$GITHUB_ENV"
168+
- uses: ./.github/actions/bench
169+
if: ${{ inputs.opt == 'all' || inputs.opt == 'opt' }}
170+
with:
171+
nix-shell: 'ci-bench'
172+
custom-shell: 'bash'
173+
nix-cache: false
174+
nix-verbose: ${{ inputs.verbose }}
175+
name: ${{ inputs.name }} (${{ inputs.compiler }})
176+
cflags: ${{ inputs.cflags }}
177+
archflags: ${{ inputs.archflags }}
178+
opt: true
179+
perf: ${{ inputs.perf }}
180+
store_results: ${{ inputs.store_results }}
181+
bench_extra_args: ${{ inputs.bench_extra_args }}
182+
gh_token: ${{ secrets.AWS_GITHUB_TOKEN }}
183+
- uses: ./.github/actions/bench
184+
if: ${{ inputs.opt == 'all' || inputs.opt == 'no_opt' }}
185+
with:
186+
nix-shell: 'ci-bench'
187+
custom-shell: 'bash'
188+
nix-cache: false
189+
nix-verbose: ${{ inputs.verbose }}
190+
name: ${{ inputs.name }} (${{ inputs.compiler }}) (no-opt)
191+
cflags: ${{ inputs.cflags }}
192+
archflags: ${{ inputs.archflags }}
193+
opt: false
194+
perf: ${{ inputs.perf }}
195+
store_results: ${{ inputs.store_results }}
196+
bench_extra_args: ${{ inputs.bench_extra_args }}
197+
gh_token: ${{ secrets.AWS_GITHUB_TOKEN }}
198+
alert_threshold: ${{ inputs.alert_threshold }}
199+
stop-ec2-runner:
200+
name: Stop ${{ inputs.name }} (${{ inputs.ec2_instance_type }})
201+
permissions:
202+
contents: 'read'
203+
id-token: 'write'
204+
needs:
205+
- start-ec2-runner
206+
- bench_nix # required to wait when the main job is done
207+
- bench_custom # required to wait when the main job is done
208+
runs-on: ubuntu-latest
209+
if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
210+
steps:
211+
- name: Configure AWS credentials
212+
uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
213+
with:
214+
role-to-assume: ${{ env.AWS_ROLE }}
215+
aws-region: ${{ inputs.aws_region }}
216+
- name: Stop EC2 runner
217+
uses: machulav/ec2-github-runner@a8c20fc0876503410b2b966c124abc2311984ce2 # v2.3.9
218+
with:
219+
mode: stop
220+
github-token: ${{ secrets.AWS_GITHUB_TOKEN }}
221+
label: ${{ needs.start-ec2-runner.outputs.label }}
222+
ec2-instance-id: ${{ needs.start-ec2-runner.outputs.ec2-instance-id }}

‎.github/workflows/ci_ec2_reusable.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@ on:
6868
env:
6969
AWS_ROLE: arn:aws:iam::904233116199:role/mldsa-native-ci
7070
AWS_REGION: us-east-1
71-
AMI_UBUNTU_LATEST_X86_64: ami-0e86e20dae9224db8
72-
AMI_UBUNTU_LATEST_AARCH64: ami-096ea6a12ea24a797
71+
AMI_UBUNTU_LATEST_X86_64: ami-084568db4383264d4
72+
AMI_UBUNTU_LATEST_AARCH64: ami-0c4e709339fa8521a
7373
jobs:
7474
start-ec2-runner:
7575
name: Start instance (${{ inputs.ec2_instance_type }})

0 commit comments

Comments
 (0)