From 1a02912c279c1f35dce2d1a3117cc34ad5ef646b Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 28 Jan 2025 16:25:10 +0100 Subject: [PATCH 1/6] wipi --- JetStreamDriver.js | 10 +++++++ package-lock.json | 10 +++---- package.json | 2 +- tests/run.mjs | 66 +++++++++++++++++++++++++++++++++++++++------- 4 files changed, 72 insertions(+), 16 deletions(-) diff --git a/JetStreamDriver.js b/JetStreamDriver.js index 766724d..c6922cf 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -279,8 +279,18 @@ class Driver { } benchmark.updateUIAfterRun(); + console.log(benchmark.name) if (isInBrowser) { + globalThis.dispatchEvent(new CustomEvent("JetStreamBenchmarkDone", { + detail: { + name: benchmark.name, + results: { + score: benchmark.score, + ...benchmark.subScores(), + } + } + })); const cache = JetStream.blobDataCache; for (const file of benchmark.plan.files) { const blobData = cache[file]; diff --git a/package-lock.json b/package-lock.json index 78422e8..2b64175 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "http-server": "^14.1.1", "local-web-server": "^5.4.0", "prettier": "^2.8.3", - "selenium-webdriver": "^4.8.0" + "selenium-webdriver": "^4.28.1" }, "engines": { "node": ">=18.13.0", @@ -3967,9 +3967,9 @@ "dev": true }, "node_modules/selenium-webdriver": { - "version": "4.27.0", - "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.27.0.tgz", - "integrity": "sha512-LkTJrNz5socxpPnWPODQ2bQ65eYx9JK+DQMYNihpTjMCqHwgWGYQnQTCAAche2W3ZP87alA+1zYPvgS8tHNzMQ==", + "version": "4.28.1", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.28.1.tgz", + "integrity": "sha512-TwbTpu/NUQkorBODGAkGowJ8sar63bvqi66/tjqhS05rBl34HkVp8DoRg1cOv2iSnNonVSbkxazS3wjbc+NRtg==", "dev": true, "funding": [ { @@ -3988,7 +3988,7 @@ "ws": "^8.18.0" }, "engines": { - "node": ">= 14.21.0" + "node": ">= 18.20.5" } }, "node_modules/semver": { diff --git a/package.json b/package.json index b3824f4..b75017c 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "eslint": "^8.38.0", "http-server": "^14.1.1", "prettier": "^2.8.3", - "selenium-webdriver": "^4.8.0", + "selenium-webdriver": "^4.28.1", "es-main":"^1.3.0", "local-web-server": "^5.4.0" } diff --git a/tests/run.mjs b/tests/run.mjs index 3001639..bfed1ff 100644 --- a/tests/run.mjs +++ b/tests/run.mjs @@ -1,7 +1,7 @@ #! /usr/bin/env node /* eslint-disable-next-line no-unused-vars */ import serve from "./server.mjs"; -import { Builder, Capabilities } from "selenium-webdriver"; +import { Builder, Capabilities, Browser, logging} from "selenium-webdriver"; import commandLineArgs from "command-line-args"; import commandLineUsage from "command-line-usage"; import assert from "assert"; @@ -64,7 +64,6 @@ switch (BROWSER) { printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`); } } - process.on("unhandledRejection", (err) => { console.error(err); process.exit(1); @@ -78,28 +77,75 @@ const PORT = options.port; const server = await serve(PORT); async function testEnd2End() { - const driver = await new Builder().withCapabilities(capabilities).build(); + var pref = new logging.Preferences(); + pref.setLevel('browser', logging.Level.ALL); + pref.setLevel('driver', logging.Level.ALL); + pref.setLevel('performance', logging.Level.ALL); + const driver = await new Builder() + .withCapabilities(capabilities) + .setLoggingPrefs(pref) + .build(); let results; try { + console.log("Preparing JetStream"); await driver.get(`http://localhost:${PORT}/index.html?worstCaseCount=2&iterationCount=3`); await driver.executeAsyncScript((callback) => { - globalThis.addEventListener("JetStreamReady", callback); + globalThis.addEventListener("JetStreamReady", () => callback()); // We might not get a chance to install the on-ready listener, thus // we also check if the runner is ready synchronously. if (globalThis?.JetStream?.isReady) callback() }); - await driver.manage().setTimeouts({ script: 60_000 }); - results = await driver.executeAsyncScript((callback) => { - globalThis.addEventListener("JetStreamDone", event => callback(event.detail)); - JetStream.start(); - }); + results = await benchmarkResults(driver) } finally { console.log("\nTests complete!"); - console.log(results) driver.quit(); server.close(); } } +async function benchmarkResults(driver) { + console.log("Starting JetStream"); + await driver.manage().setTimeouts({ script: 60_000 }); + await driver.executeScript(() => { + globalThis.JetStreamDone = false; + globalThis.JetStreamResults = []; + globalThis.addEventListener("JetStreamDone", event => { + globalThis.JetStreamDone = true; + }); + globalThis.addEventListener("JetStreamBenchmarkDone", event => { + globalThis.JetStreamResults.push(event.detail); + }); + JetStream.start(); + }); + + await new Promise(resolve => pollIncrementalResults(driver, resolve)); + const resultString = await driver.executeScript(() => { + if (globalThis.JetStreamDone) + return JSON.stringify(JetStream.resultsObject()); + }); + return JSON.parse(resultString); +} + +const UPDATE_INTERVAL = 250; +async function pollIncrementalResults(driver, resolve) { + const internalId = setInterval(async function logResult() { + const {done, results} = await driver.executeAsyncScript((callback) => { + callback({ + done: globalThis.JetStreamDone, + results: JSON.stringify(globalThis.JetStreamResults.splice(0, Infinity)) + }); + }); + JSON.parse(results).forEach(logBenchmarkResult); + if (done) { + clearInterval(internalId); + resolve() + } + }, UPDATE_INTERVAL) +} + +function logBenchmarkResult(benchmarkResult) { + console.log(benchmarkResult.name, benchmarkResult.results) +} + setImmediate(testEnd2End); From 4634f88d578420f56e7dcf62f021bf3126262356 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 28 Jan 2025 16:25:57 +0100 Subject: [PATCH 2/6] reverti --- package-lock.json | 10 +++++----- package.json | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2b64175..78422e8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "http-server": "^14.1.1", "local-web-server": "^5.4.0", "prettier": "^2.8.3", - "selenium-webdriver": "^4.28.1" + "selenium-webdriver": "^4.8.0" }, "engines": { "node": ">=18.13.0", @@ -3967,9 +3967,9 @@ "dev": true }, "node_modules/selenium-webdriver": { - "version": "4.28.1", - "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.28.1.tgz", - "integrity": "sha512-TwbTpu/NUQkorBODGAkGowJ8sar63bvqi66/tjqhS05rBl34HkVp8DoRg1cOv2iSnNonVSbkxazS3wjbc+NRtg==", + "version": "4.27.0", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.27.0.tgz", + "integrity": "sha512-LkTJrNz5socxpPnWPODQ2bQ65eYx9JK+DQMYNihpTjMCqHwgWGYQnQTCAAche2W3ZP87alA+1zYPvgS8tHNzMQ==", "dev": true, "funding": [ { @@ -3988,7 +3988,7 @@ "ws": "^8.18.0" }, "engines": { - "node": ">= 18.20.5" + "node": ">= 14.21.0" } }, "node_modules/semver": { diff --git a/package.json b/package.json index b75017c..b3824f4 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "eslint": "^8.38.0", "http-server": "^14.1.1", "prettier": "^2.8.3", - "selenium-webdriver": "^4.28.1", + "selenium-webdriver": "^4.8.0", "es-main":"^1.3.0", "local-web-server": "^5.4.0" } From 10ba50a63ce31bc0cf6982d7f00d9d6cd187bcbf Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 28 Jan 2025 16:29:30 +0100 Subject: [PATCH 3/6] cleanup --- tests/run.mjs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/run.mjs b/tests/run.mjs index bfed1ff..68e7469 100644 --- a/tests/run.mjs +++ b/tests/run.mjs @@ -1,10 +1,9 @@ #! /usr/bin/env node /* eslint-disable-next-line no-unused-vars */ import serve from "./server.mjs"; -import { Builder, Capabilities, Browser, logging} from "selenium-webdriver"; +import { Builder, Capabilities} from "selenium-webdriver"; import commandLineArgs from "command-line-args"; import commandLineUsage from "command-line-usage"; -import assert from "assert"; const optionDefinitions = [ { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome]. By default the $BROWSER env variable is used." }, @@ -77,13 +76,8 @@ const PORT = options.port; const server = await serve(PORT); async function testEnd2End() { - var pref = new logging.Preferences(); - pref.setLevel('browser', logging.Level.ALL); - pref.setLevel('driver', logging.Level.ALL); - pref.setLevel('performance', logging.Level.ALL); const driver = await new Builder() .withCapabilities(capabilities) - .setLoggingPrefs(pref) .build(); let results; try { From 5fd1b6fff76a8e1d524ff85bb5e6aeb309039034 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 28 Jan 2025 16:32:45 +0100 Subject: [PATCH 4/6] further cleanup --- tests/run.mjs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/run.mjs b/tests/run.mjs index 68e7469..0c8395d 100644 --- a/tests/run.mjs +++ b/tests/run.mjs @@ -1,7 +1,7 @@ #! /usr/bin/env node /* eslint-disable-next-line no-unused-vars */ import serve from "./server.mjs"; -import { Builder, Capabilities} from "selenium-webdriver"; +import { Builder, Capabilities } from "selenium-webdriver"; import commandLineArgs from "command-line-args"; import commandLineUsage from "command-line-usage"; @@ -63,6 +63,7 @@ switch (BROWSER) { printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`); } } + process.on("unhandledRejection", (err) => { console.error(err); process.exit(1); @@ -76,9 +77,7 @@ const PORT = options.port; const server = await serve(PORT); async function testEnd2End() { - const driver = await new Builder() - .withCapabilities(capabilities) - .build(); + const driver = await new Builder().withCapabilities(capabilities).build(); let results; try { console.log("Preparing JetStream"); @@ -90,7 +89,8 @@ async function testEnd2End() { if (globalThis?.JetStream?.isReady) callback() }); - results = await benchmarkResults(driver) + results = await benchmarkResults(driver); + // FIXME: validate results; } finally { console.log("\nTests complete!"); driver.quit(); @@ -112,33 +112,31 @@ async function benchmarkResults(driver) { }); JetStream.start(); }); - await new Promise(resolve => pollIncrementalResults(driver, resolve)); const resultString = await driver.executeScript(() => { - if (globalThis.JetStreamDone) - return JSON.stringify(JetStream.resultsObject()); + return JSON.stringify(JetStream.resultsObject()); }); return JSON.parse(resultString); } const UPDATE_INTERVAL = 250; async function pollIncrementalResults(driver, resolve) { - const internalId = setInterval(async function logResult() { + const intervalId = setInterval(async function logResult() { const {done, results} = await driver.executeAsyncScript((callback) => { callback({ done: globalThis.JetStreamDone, results: JSON.stringify(globalThis.JetStreamResults.splice(0, Infinity)) }); }); - JSON.parse(results).forEach(logBenchmarkResult); + JSON.parse(results).forEach(logIncrementalResult); if (done) { - clearInterval(internalId); + clearInterval(intervalId); resolve() } }, UPDATE_INTERVAL) } -function logBenchmarkResult(benchmarkResult) { +function logIncrementalResult(benchmarkResult) { console.log(benchmarkResult.name, benchmarkResult.results) } From 6f59500f07987dab229338d724b19c236732329b Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 28 Jan 2025 16:33:32 +0100 Subject: [PATCH 5/6] more cleanup --- tests/run.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/run.mjs b/tests/run.mjs index 0c8395d..8bdd5f9 100644 --- a/tests/run.mjs +++ b/tests/run.mjs @@ -125,8 +125,8 @@ async function pollIncrementalResults(driver, resolve) { const {done, results} = await driver.executeAsyncScript((callback) => { callback({ done: globalThis.JetStreamDone, - results: JSON.stringify(globalThis.JetStreamResults.splice(0, Infinity)) - }); + results: JSON.stringify(globalThis.JetStreamResults.splice(0, Infinity)), + }); }); JSON.parse(results).forEach(logIncrementalResult); if (done) { From 4629828c9f3b663e00ae5271b88603b0c4e695df Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 28 Jan 2025 17:05:11 +0100 Subject: [PATCH 6/6] cleanup --- JetStreamDriver.js | 23 +++++++++++++++-------- tests/run.mjs | 32 +++++++++++++------------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/JetStreamDriver.js b/JetStreamDriver.js index c6922cf..2478e91 100644 --- a/JetStreamDriver.js +++ b/JetStreamDriver.js @@ -239,6 +239,8 @@ const fileLoader = (function() { class Driver { constructor() { this.isReady = false; + this.isDone = false; + this.incrementalResults = []; this.benchmarks = []; this.blobDataCache = { }; this.loadCache = { }; @@ -282,15 +284,13 @@ class Driver { console.log(benchmark.name) if (isInBrowser) { - globalThis.dispatchEvent(new CustomEvent("JetStreamBenchmarkDone", { - detail: { - name: benchmark.name, - results: { - score: benchmark.score, - ...benchmark.subScores(), - } + this.incrementalResults.push({ + name: benchmark.name, + results: { + Score: benchmark.score, + ...benchmark.subScores(), } - })); + }); const cache = JetStream.blobDataCache; for (const file of benchmark.plan.files) { const blobData = cache[file]; @@ -344,6 +344,7 @@ class Driver { this.reportScoreToRunBenchmarkRunner(); this.dumpJSONResultsIfNeeded(); + this.isDone = true; if (isInBrowser) { globalThis.dispatchEvent(new CustomEvent("JetStreamDone", { detail: this.resultsObject() @@ -351,6 +352,12 @@ class Driver { } } + drainIncrementalResults() { + const currentIncrementalResults = this.incrementalResults; + this.incrementalResults = []; + return currentIncrementalResults + } + runCode(string) { if (!isInBrowser) { diff --git a/tests/run.mjs b/tests/run.mjs index 8bdd5f9..7ed4109 100644 --- a/tests/run.mjs +++ b/tests/run.mjs @@ -80,8 +80,9 @@ async function testEnd2End() { const driver = await new Builder().withCapabilities(capabilities).build(); let results; try { - console.log("Preparing JetStream"); - await driver.get(`http://localhost:${PORT}/index.html?worstCaseCount=2&iterationCount=3`); + const url = `http://localhost:${PORT}/index.html?worstCaseCount=2&iterationCount=3`; + console.log(`JetStream PREPARE ${url}`); + await driver.get(url); await driver.executeAsyncScript((callback) => { globalThis.addEventListener("JetStreamReady", () => callback()); // We might not get a chance to install the on-ready listener, thus @@ -99,22 +100,15 @@ async function testEnd2End() { } async function benchmarkResults(driver) { - console.log("Starting JetStream"); + console.log("JetStream START"); await driver.manage().setTimeouts({ script: 60_000 }); - await driver.executeScript(() => { - globalThis.JetStreamDone = false; - globalThis.JetStreamResults = []; - globalThis.addEventListener("JetStreamDone", event => { - globalThis.JetStreamDone = true; - }); - globalThis.addEventListener("JetStreamBenchmarkDone", event => { - globalThis.JetStreamResults.push(event.detail); - }); - JetStream.start(); + await driver.executeAsyncScript((callback) => { + globalThis.JetStream.start(); + callback(); }); await new Promise(resolve => pollIncrementalResults(driver, resolve)); const resultString = await driver.executeScript(() => { - return JSON.stringify(JetStream.resultsObject()); + return JSON.stringify(globalThis.JetStream.resultsObject()); }); return JSON.parse(resultString); } @@ -122,11 +116,11 @@ async function benchmarkResults(driver) { const UPDATE_INTERVAL = 250; async function pollIncrementalResults(driver, resolve) { const intervalId = setInterval(async function logResult() { - const {done, results} = await driver.executeAsyncScript((callback) => { - callback({ - done: globalThis.JetStreamDone, - results: JSON.stringify(globalThis.JetStreamResults.splice(0, Infinity)), - }); + const {done, results} = await driver.executeScript(() => { + return { + done: globalThis.JetStream.isDone, + results: JSON.stringify(globalThis.JetStream.drainIncrementalResults()), + }; }); JSON.parse(results).forEach(logIncrementalResult); if (done) {