Skip to content

Commit 0fcd811

Browse files
ComradeProgrammerComradeProgrammer
committed
add github action for running minikube-image-benchmark
1 parent 8cdc5c7 commit 0fcd811

File tree

5 files changed

+373
-0
lines changed

5 files changed

+373
-0
lines changed

‎.github/workflows/minikube-image-benchmark.yml

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: "publish image benchmark"
2+
on:
3+
workflow_dispatch:
4+
schedule:
5+
# every day at 7am & 7pm pacific
6+
- cron: "0 2,14 * * *"
7+
env:
8+
GOPROXY: https://proxy.golang.org
9+
GO_VERSION: '1.20.6'
10+
permissions:
11+
contents: read
12+
13+
jobs:
14+
image-benchmark:
15+
if: github.repository == 'kubernetes/minikube'
16+
runs-on: ubuntu-20.04
17+
env:
18+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
19+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
20+
AWS_DEFAULT_REGION: 'us-west-1'
21+
steps:
22+
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
23+
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753
24+
with:
25+
go-version: ${{env.GO_VERSION}}
26+
cache-dependency-path: ./go.sum
27+
- name: Run Benchmark
28+
run: |
29+
./hack/benchmark/image-build/publish-chart.sh
30+
31+

‎.gitmodules

+3
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,6 @@
44
[submodule "hack/benchmark/time-to-k8s/time-to-k8s-repo"]
55
path = hack/benchmark/time-to-k8s/time-to-k8s-repo
66
url = https://github.com/tstromberg/time-to-k8s.git
7+
[submodule "hack/benchmark/image-build/minikube-image-benchmark"]
8+
path = hack/benchmark/image-build/minikube-image-benchmark
9+
url = https://github.com/GoogleContainerTools/minikube-image-benchmark.git

‎hack/benchmark/image-build/generate-chart.go

+280
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,280 @@
1+
/*
2+
Copyright 2023 The Kubernetes Authors All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package main
18+
19+
import (
20+
"encoding/csv"
21+
"encoding/json"
22+
"flag"
23+
"fmt"
24+
"image/color"
25+
"io"
26+
"log"
27+
"math"
28+
"os"
29+
"path/filepath"
30+
"strconv"
31+
"time"
32+
33+
"gonum.org/v1/plot"
34+
"gonum.org/v1/plot/plotter"
35+
"gonum.org/v1/plot/plotutil"
36+
"gonum.org/v1/plot/vg"
37+
"gonum.org/v1/plot/vg/draw"
38+
)
39+
40+
var Images = []string{
41+
"buildpacksFewLargeFiles",
42+
// to simplify the output, the following images are omitted
43+
// "buildpacksFewSmallFiles",
44+
// "buildpacksManyLargeFiles",
45+
// "buildpacksManySmallFiles",
46+
}
47+
48+
var Environments = []string{
49+
"MinikubeImageLoadDocker",
50+
"MinikubeImageBuild",
51+
"MinikubeDockerEnvDocker",
52+
"MinikubeAddonRegistryDocker",
53+
"MinikubeImageLoadContainerd",
54+
"MinikubeImageContainerd",
55+
"MinikubeAddonRegistryContainerd",
56+
"MinikubeImageLoadCrio",
57+
"MinikubeImageCrio",
58+
"MinikubeAddonRegistryCrio",
59+
"Kind",
60+
"K3d",
61+
"Microk8s",
62+
}
63+
64+
var RuntimeEnvironments = map[string][]string{
65+
"docker": {
66+
"MinikubeImageLoadDocker",
67+
"MinikubeImageBuild",
68+
"MinikubeDockerEnvDocker",
69+
"MinikubeAddonRegistryDocker",
70+
},
71+
72+
"containerd": {
73+
"MinikubeImageLoadContainerd",
74+
"MinikubeImageContainerd",
75+
"MinikubeAddonRegistryContainerd",
76+
},
77+
}
78+
79+
const (
80+
INTERATIVE = "Iterative"
81+
NONINTERATIVE = "NonIterative"
82+
)
83+
84+
var Methods = []string{
85+
INTERATIVE,
86+
// to simplify the output, non-interative is omitted
87+
// NONINTERATIVE,
88+
}
89+
90+
// env name-> test result
91+
type TestResult map[string]float64
92+
93+
func NewTestResult(values []float64) TestResult {
94+
res := make(TestResult)
95+
for index, v := range values {
96+
res[Environments[index]] = v
97+
}
98+
return res
99+
}
100+
101+
// imageName->TestResult
102+
type ImageTestResults map[string]TestResult
103+
104+
type MethodTestResults struct {
105+
Date time.Time
106+
// method name -> results
107+
Results map[string]ImageTestResults
108+
}
109+
110+
type Records struct {
111+
Records []MethodTestResults
112+
}
113+
114+
func main() {
115+
latestTestResultPath := flag.String("csv", "", "path to the CSV file containing the latest benchmark result")
116+
pastTestRecordsPath := flag.String("past-runs", "", "path to the JSON file containing the past benchmark results")
117+
chartsPath := flag.String("charts", "", "path to the folder to write the daily charts to")
118+
flag.Parse()
119+
120+
latestBenchmark := readInLatestTestResult(*latestTestResultPath)
121+
latestBenchmark.Date = time.Now()
122+
pastBenchmarks := readInPastTestResults(*pastTestRecordsPath)
123+
pastBenchmarks.Records = append(pastBenchmarks.Records, latestBenchmark)
124+
updatePastTestResults(pastBenchmarks, *pastTestRecordsPath)
125+
createDailyChart(pastBenchmarks, *chartsPath)
126+
}
127+
128+
// readInLatestTestResult reads in the latest benchmark result from a CSV file
129+
// and return the MethodTestResults object
130+
func readInLatestTestResult(latestBenchmarkPath string) MethodTestResults {
131+
132+
var res = MethodTestResults{
133+
Results: make(map[string]ImageTestResults),
134+
}
135+
res.Results[INTERATIVE] = make(ImageTestResults)
136+
res.Results[NONINTERATIVE] = make(ImageTestResults)
137+
138+
f, err := os.Open(latestBenchmarkPath)
139+
if err != nil {
140+
log.Fatal(err)
141+
}
142+
143+
r := csv.NewReader(f)
144+
for {
145+
line, err := r.Read()
146+
if err == io.EOF {
147+
break
148+
}
149+
if err != nil {
150+
log.Fatal(err)
151+
}
152+
153+
// skip the first line of the CSV file
154+
if line[0] == "image" {
155+
continue
156+
}
157+
158+
valuesInterative := []float64{}
159+
valuesNonInterative := []float64{}
160+
// interative test results of each env are stored in the following columns
161+
indicesInterative := []int{1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49}
162+
// non-interative test results of each env are stored in the following columns
163+
indicesNonInterative := []int{3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51}
164+
165+
for _, i := range indicesInterative {
166+
v, err := strconv.ParseFloat(line[i], 64)
167+
if err != nil {
168+
log.Fatal(err)
169+
}
170+
valuesInterative = append(valuesInterative, v)
171+
}
172+
173+
for _, i := range indicesNonInterative {
174+
v, err := strconv.ParseFloat(line[i], 64)
175+
if err != nil {
176+
log.Fatal(err)
177+
}
178+
valuesNonInterative = append(valuesNonInterative, v)
179+
}
180+
181+
imageName := line[0]
182+
183+
res.Results[INTERATIVE][imageName] = NewTestResult(valuesInterative)
184+
res.Results[NONINTERATIVE][imageName] = NewTestResult(valuesNonInterative)
185+
186+
}
187+
188+
return res
189+
}
190+
191+
// readInPastTestResults reads in the past benchmark results from a JSON file
192+
func readInPastTestResults(pastTestRecordPath string) Records {
193+
194+
record := Records{}
195+
data, err := os.ReadFile(pastTestRecordPath)
196+
if os.IsNotExist(err) {
197+
return record
198+
}
199+
if err != nil {
200+
log.Fatal(err)
201+
}
202+
203+
if err := json.Unmarshal(data, &record); err != nil {
204+
log.Fatal(err)
205+
}
206+
207+
return record
208+
}
209+
210+
// updateRunsFile overwrites the run file with the updated benchmarks list
211+
func updatePastTestResults(h Records, pastTestRecordPath string) {
212+
b, err := json.Marshal(h)
213+
if err != nil {
214+
log.Fatal(err)
215+
}
216+
217+
if err := os.WriteFile(pastTestRecordPath, b, 0600); err != nil {
218+
log.Fatal(err)
219+
}
220+
}
221+
func createDailyChart(record Records, outputFolder string) {
222+
223+
for _, method := range Methods {
224+
for _, image := range Images {
225+
createChart(record, method, image, "docker", outputFolder)
226+
createChart(record, method, image, "containerd", outputFolder)
227+
}
228+
}
229+
}
230+
231+
func createChart(record Records, methodName string, imageName string, runtime string, chartOutputPath string) {
232+
p := plot.New()
233+
p.Add(plotter.NewGrid())
234+
p.Legend.Top = true
235+
p.Title.Text = fmt.Sprintf("%s-%s-%s-performance", methodName, imageName, runtime)
236+
p.X.Label.Text = "date"
237+
p.X.Tick.Marker = plot.TimeTicks{Format: "2006-01-02"}
238+
p.Y.Label.Text = "time (seconds)"
239+
yMaxTotal := float64(0)
240+
241+
// gonum plot do not have enough default colors in any group
242+
// so we combine different group of default colors
243+
colors := append([]color.Color{}, plotutil.SoftColors...)
244+
colors = append(colors, plotutil.DarkColors...)
245+
246+
pointGroup := make(map[string]plotter.XYs)
247+
for _, name := range RuntimeEnvironments[runtime] {
248+
pointGroup[name] = make(plotter.XYs, len(record.Records))
249+
250+
}
251+
252+
for i := 0; i < len(record.Records); i++ {
253+
for _, envName := range RuntimeEnvironments[runtime] {
254+
pointGroup[envName][i].X = float64(record.Records[i].Date.Unix())
255+
pointGroup[envName][i].Y = record.Records[i].Results[methodName][imageName][envName]
256+
yMaxTotal = math.Max(yMaxTotal, pointGroup[envName][i].Y)
257+
}
258+
}
259+
p.Y.Max = yMaxTotal
260+
261+
i := 0
262+
for envName, xys := range pointGroup {
263+
line, points, err := plotter.NewLinePoints(xys)
264+
if err != nil {
265+
log.Fatal(err)
266+
}
267+
line.Color = colors[i]
268+
points.Color = colors[i]
269+
points.Shape = draw.CircleGlyph{}
270+
i++
271+
p.Add(line, points)
272+
p.Legend.Add(envName, line)
273+
}
274+
275+
filename := filepath.Join(chartOutputPath, fmt.Sprintf("%s_%s_%s_chart.png", methodName, imageName, runtime))
276+
277+
if err := p.Save(12*vg.Inch, 8*vg.Inch, filename); err != nil {
278+
log.Fatalf("failed creating png: %v", err)
279+
}
280+
}

‎hack/benchmark/image-build/minikube-image-benchmark

‎hack/benchmark/image-build/publish-chart.sh

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash
2+
3+
# Copyright 2023 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -x
18+
19+
BUCKET="s3://image-benchmark"
20+
21+
install_minikube() {
22+
make
23+
sudo install ./out/minikube /usr/local/bin/minikube
24+
}
25+
26+
run_benchmark() {
27+
( cd ./hack/benchmark/image-build/minikube-image-benchmark &&
28+
git submodule update --init &&
29+
make &&
30+
./out/benchmark )
31+
}
32+
33+
generate_chart() {
34+
go run ./hack/benchmark/image-build/generate-chart.go --csv hack/benchmark/image-build/minikube-image-benchmark/out/results.csv --past-runs record.json
35+
}
36+
37+
copy() {
38+
aws s3 cp "$1" "$2"
39+
}
40+
41+
cleanup() {
42+
rm ./Iterative_buildpacksFewLargeFiles_containerd_chart.png
43+
rm ./Iterative_buildpacksFewLargeFiles_docker_chart.png
44+
rm hack/benchmark/image-build/minikube-image-benchmark/out/results.csv
45+
}
46+
47+
48+
install_minikube
49+
copy "$BUCKET/record.json" ./record.json
50+
set -e
51+
52+
run_benchmark
53+
generate_chart
54+
55+
copy ./record.json "$BUCKET/record.json"
56+
copy ./Iterative_buildpacksFewLargeFiles_containerd_chart.png "$BUCKET/Iterative_buildpacksFewLargeFiles_containerd_chart.png"
57+
copy ./Iterative_buildpacksFewLargeFiles_docker_chart.png "$BUCKET/Iterative_buildpacksFewLargeFiles_docker_chart.png"
58+
cleanup

0 commit comments

Comments
 (0)