Skip to content

Commit ad89a11

Browse files
sokrasokra
committed
add report
1 parent 2c2af0d commit ad89a11

File tree

5 files changed

+119
-25
lines changed

5 files changed

+119
-25
lines changed

‎.github/workflows/compare.yml

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,12 @@ jobs:
2323
- "${{ github.event.inputs.current }}"
2424
case:
2525
- minimal
26-
# - esbuild-three
26+
- esbuild-three
2727
scenario:
2828
- development-default-build
2929
- development-cached-build
30-
# - development-cached-pnp-build
31-
# - development-cached-rebuild
30+
- development-cached-pnp-build
31+
- development-cached-rebuild
3232
# - production-default-build
3333
# - production-cached-build
3434
# - production-source-map-build
@@ -59,6 +59,7 @@ jobs:
5959
needs: [bench]
6060
runs-on: ubuntu-latest
6161
steps:
62+
- run: echo ${{ github.event.inputs.base }} vs ${{ github.event.inputs.current }}
6263
- uses: actions/checkout@v2
6364
- name: Use Node.js
6465
uses: actions/setup-node@v1
@@ -68,4 +69,4 @@ jobs:
6869
with:
6970
name: compare-results
7071
path: output
71-
- run: ls -l output
72+
- run: node bin/report.js

‎.prettierrc.js renamed to ‎.prettierrc.cjs

File renamed without changes.

‎bin/compare.js

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ const rootDir = resolve(fileURLToPath(import.meta.url), "../..");
1717

1818
(async () => {
1919
const diff = await compare(caseName, scenarioName, {
20-
runs: 20,
20+
runs: 30,
2121
verboseSetup: true,
2222
baselineDependencies: {
2323
webpack: `webpack/webpack#${baseline}`,
@@ -26,10 +26,10 @@ const rootDir = resolve(fileURLToPath(import.meta.url), "../..");
2626
webpack: `webpack/webpack#${current}`,
2727
},
2828
});
29-
console.log(formatDiffTable(diff, true));
29+
console.log(formatDiffTable(diff, { colors: true, verbose: true }));
3030
await mkdir(resolve(rootDir, "output"), { recursive: true });
3131
await writeFile(
32-
resolve(rootDir, `output/${caseName}-${scenarioName}.json`),
32+
resolve(rootDir, `output/${caseName}_${scenarioName}.json`),
3333
JSON.stringify(diff, null, 2)
3434
);
3535
})().catch((err) => {

‎bin/report.js

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import { readdir, readFile, writeFile } from "fs/promises";
2+
import { resolve } from "path";
3+
import { fileURLToPath } from "url";
4+
import { formatDiffTable } from "../lib/utils.js";
5+
6+
const rootDir = resolve(fileURLToPath(import.meta.url), "../..");
7+
8+
(async () => {
9+
const diffNames = await readdir(resolve(rootDir, "output"));
10+
const diffs = await Promise.all(
11+
diffNames.map(async (name) => ({
12+
case: name.split("_")[0],
13+
scenario: name.split("_")[1].replace(".json", ""),
14+
diff: JSON.parse(
15+
await readFile(resolve(rootDir, "output", name), "utf-8")
16+
),
17+
}))
18+
);
19+
20+
const mergedDiff = {};
21+
for (const { case: caseName, scenario, diff } of diffs) {
22+
for (const key of Object.keys(diff)) {
23+
mergedDiff[`${caseName} ${scenario} ${key}`] = diff[key];
24+
}
25+
}
26+
27+
console.log(formatDiffTable(mergedDiff, { colors: true }));
28+
})().catch((err) => {
29+
process.exitCode = 1;
30+
console.error(err.stack);
31+
});

‎lib/utils.js

Lines changed: 80 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,51 @@ export const clearCaches = async (directory) => {
8181
);
8282
};
8383

84+
const T_TABLE = [
85+
12.71,
86+
4.303,
87+
3.182,
88+
2.776,
89+
2.571,
90+
2.447,
91+
2.365,
92+
2.306,
93+
2.262,
94+
2.228,
95+
2.201,
96+
2.179,
97+
2.16,
98+
2.145,
99+
2.131,
100+
2.12,
101+
2.11,
102+
2.101,
103+
2.093,
104+
2.086,
105+
2.08,
106+
2.074,
107+
2.069,
108+
2.064,
109+
2.06,
110+
2.056,
111+
2.052,
112+
2.048,
113+
2.045,
114+
2.042,
115+
];
116+
117+
const tDist95Two = (n) => {
118+
if (n <= 1) return 12.71;
119+
if (n <= 30) return T_TABLE[n - 1];
120+
if (n <= 40) return 2.021;
121+
if (n <= 50) return 2.009;
122+
if (n <= 60) return 2.0;
123+
if (n <= 80) return 1.99;
124+
if (n <= 100) return 1.984;
125+
if (n <= 120) return 1.98;
126+
return 1.96;
127+
};
128+
84129
export const calcStatistics = (results) => {
85130
const stats = {};
86131
for (const key of Object.keys(results[0])) {
@@ -93,6 +138,8 @@ export const calcStatistics = (results) => {
93138
const variance =
94139
values.reduce((sum, v) => sum + (mean - v) ** 2, 0) / values.length;
95140
const stdDev = Math.sqrt(variance);
141+
const confidence =
142+
(tDist95Two(values.length - 1) * stdDev) / Math.sqrt(values.length);
96143
stats[key] = {
97144
min: Math.min(...values),
98145
max: Math.max(...values),
@@ -103,8 +150,9 @@ export const calcStatistics = (results) => {
103150
: values[(values.length - 1) / 2],
104151
variance,
105152
stdDev,
106-
low: mean - stdDev,
107-
high: mean + stdDev,
153+
confidence: confidence * 2,
154+
low: mean - confidence,
155+
high: mean + confidence,
108156
count: values.length,
109157
};
110158
}
@@ -123,9 +171,6 @@ export const compareStatistics = (
123171
const currentValue = current[key];
124172
if (baseValue === undefined || currentValue === undefined) continue;
125173
if ("mean" in baseValue) {
126-
const getDiff = (b, c) => {
127-
return c / b;
128-
};
129174
diff[key] = {
130175
relevance: currentValue.mean / total,
131176
};
@@ -136,20 +181,34 @@ export const compareStatistics = (
136181
diff[key].highLow = currentValue.low / baseValue.high;
137182
diff[key].baseStdDev = baseValue.stdDev / baseValue.mean;
138183
diff[key].currentStdDev = currentValue.stdDev / currentValue.mean;
184+
diff[key].baseConfidence = baseValue.confidence / baseValue.mean;
185+
diff[key].currentConfidence = currentValue.confidence / currentValue.mean;
139186
} else {
140187
diff[key] = compareStatistics(baseValue, currentValue, total);
141188
}
142189
}
143190
return diff;
144191
};
145192

146-
export const formatDiffTable = (diff, colors) => {
147-
const entries = Object.keys(diff).map((key) => ({ name: key, ...diff[key] }));
193+
export const formatDiffTable = (
194+
diff,
195+
{ colors, verbose, limit, threshold }
196+
) => {
197+
let entries = Object.keys(diff).map((key) => ({ name: key, ...diff[key] }));
148198
entries.sort((a, b) => b.relevance - a.relevance);
199+
if (!verbose) {
200+
entries = entries.filter((e) => e.lowHigh < 1 || e.highLow > 1);
201+
}
202+
if (threshold) {
203+
entries = entries.filter((e) => e.relevance >= threshold);
204+
}
205+
if (limit) {
206+
entries = entries.slice(0, limit);
207+
}
149208
const offset = (factor) => {
150209
if (factor > 10) return `${Math.round(factor * 10) / 10}x`;
151-
if (factor > 1.1) return `+${Math.round(100 - factor * 100)}%`;
152-
if (factor > 1) return `+${Math.round(1000 - factor * 1000) / 10}%`;
210+
if (factor > 1.1) return `+${Math.round(factor * 100 - 100)}%`;
211+
if (factor > 1) return `+${Math.round(factor * 1000 - 1000) / 10}%`;
153212
if (factor > 0.9) return `-${Math.round(1000 - factor * 1000) / 10}%`;
154213
return `-${Math.round(100 - factor * 100)}%`;
155214
};
@@ -169,14 +228,17 @@ export const formatDiffTable = (diff, colors) => {
169228
: "unclear",
170229
name: (l) => l.name,
171230
mean: (l) => offset(l.mean),
172-
med: (l) => offset(l.median),
173-
min: (l) => offset(l.min),
174-
low: (l) => offset(l.low),
175-
high: (l) => offset(l.high),
176-
max: (l) => offset(l.max),
177-
std: (l) => offset(l.stdDev),
178-
baseStd: (l) => percentage(l.baseStdDev),
179-
curStd: (l) => percentage(l.currentStdDev),
231+
...(verbose
232+
? {
233+
med: (l) => offset(l.median),
234+
min: (l) => offset(l.min),
235+
max: (l) => offset(l.max),
236+
std: (l) => offset(l.stdDev),
237+
con: (l) => offset(l.confidence),
238+
}
239+
: undefined),
240+
baseCon: (l) => percentage(l.baseConfidence),
241+
curCon: (l) => percentage(l.currentConfidence),
180242
};
181243
const rows = entries.map((entry) =>
182244
Object.keys(columns).map((key) => columns[key](entry) || "")
@@ -195,7 +257,7 @@ export const formatDiffTable = (diff, colors) => {
195257
...rows.map((row) => {
196258
const line = getLine(row);
197259
if (colors) {
198-
if (row[1].startsWith("+"))
260+
if (row[1].startsWith("+") || row[1].endsWith("x"))
199261
return `\u001b[1m\u001b[31m${line}\u001b[39m\u001b[22m`;
200262
if (row[1].startsWith("-"))
201263
return `\u001b[1m\u001b[32m${line}\u001b[39m\u001b[22m`;

0 commit comments

Comments
 (0)