test(NODE-6756): add tags to benchmarks (mongodb#751)

Co-authored-by: Neal Beeken <[email protected]>

Author: W-A-James

.evergreen/config.yml

@@ -40,6 +40,15 @@ functions:
         args:
           - .evergreen/install-dependencies.sh
 
+  perf send:
+      - command: subprocess.exec
+        params:
+          working_dir: src
+          binary: bash
+          add_expansions_to_env: true
+          args:
+            - .evergreen/perf-send.sh
+
   run tests:
     - command: subprocess.exec
       type: test
@@ -257,9 +266,9 @@ tasks:
         vars:
           WARMUP: 1000
           ITERATIONS: 1000
-      - command: perf.send
-        params:
-          file: src/test/bench/etc/resultsCollectedMeans.json
+      - func: perf send
+        vars:
+          TARGET_FILE: ./test/bench/etc/resultsCollectedMeans.json
   - name: run-custom-benchmarks
     commands:
       - func: fetch source
@@ -271,9 +280,9 @@ tasks:
         vars:
           NPM_VERSION: 9
       - func: run custom benchmarks
-      - command: perf.send
-        params:
-          file: src/customBenchmarkResults.json
+      - func: perf send
+        vars:
+          TARGET_FILE: ./customBenchmarkResults.json
   - name: run-spec-benchmarks
     commands:
       - func: fetch source
@@ -285,9 +294,9 @@ tasks:
         vars:
           NPM_VERSION: 9
       - func: run spec benchmarks
-      - command: perf.send
-        params:
-          file: src/bsonBench.json
+      - func: perf send
+        vars:
+          TARGET_FILE: ./bsonBench.json
   - name: check-eslint-plugin
     commands:
       - func: fetch source

.evergreen/perf-send.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh
+set -o xtrace
+TARGET_FILE=$TARGET_FILE
+
+node ./.evergreen/perf_send.mjs $TARGET_FILE

.evergreen/perf_send.mjs

@@ -0,0 +1,73 @@
+import fs from 'fs/promises';
+import util from 'util';
+
+const API_PATH = 'https://performance-monitoring-api.corp.mongodb.com/raw_perf_results';
+
+const resultFile = process.argv[2];
+if (resultFile == null) {
+  throw new Error('Must specify result file');
+}
+
+// Get expansions
+const {
+  execution,
+  requester,
+  project,
+  task_id,
+  task_name,
+  revision_order_id,
+  build_variant: variant,
+  version_id: version
+} = process.env;
+
+const orderSplit = revision_order_id?.split('_');
+const order = Number(orderSplit ? orderSplit[orderSplit.length - 1] : undefined);
+
+if (!Number.isInteger(order)) throw new Error(`Failed to parse integer from order, revision_order_id=${revision_order_id}`);
+
+const results = JSON.parse(await fs.readFile(resultFile, 'utf8'));
+
+// FIXME(NODE-6838): We are using dummy dates here just to be able to successfully post our results
+for (const r of results) {
+  r.created_at = new Date().toISOString();
+  r.completed_at = new Date().toISOString();
+}
+
+const body = {
+  id: {
+    project,
+    version,
+    variant,
+    order,
+    task_name,
+    task_id,
+    execution,
+    mainline: requester === 'commit'
+  },
+  results
+};
+
+console.log('POST', util.inspect(body, { depth: Infinity }));
+
+const resp = await fetch(API_PATH, {
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json',
+    accept: 'application/json'
+  },
+  body: JSON.stringify(body)
+});
+
+const responseText = await resp.text();
+let jsonResponse = null;
+try {
+  jsonResponse = JSON.parse(responseText)
+} catch (cause) {
+  console.log('Failed to parse json response', cause);
+}
+
+console.log(resp.statusText, util.inspect(jsonResponse ?? responseText, { depth: Infinity }));
+
+if (jsonResponse.message == null) throw new Error("Didn't get success message");
+
+console.log(jsonResponse.message);

package-lock.json

@@ -40,7 +40,7 @@
     "benchmark": "^2.1.4",
     "chai": "^4.4.1",
     "chalk": "^5.3.0",
-    "dbx-js-tools": "github:mongodb-js/dbx-js-tools",
+    "dbx-js-tools": "github:mongodb-js/dbx-js-tools#main",
     "eslint": "^9.8.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-no-bigint-usage": "file:etc/eslint/no-bigint-usage",
@@ -102,9 +102,10 @@
     "check:tsd": "npm run build:dts && tsd",
     "check:web": "WEB=true mocha test/node",
     "check:web-no-bigint": "WEB=true NO_BIGINT=true mocha test/node",
-    "check:granular-bench": "npm run build:bench && node ./test/bench/etc/run_granular_benchmarks.js",
-    "check:spec-bench": "npm run build:bench && node ./test/bench/lib/spec/bsonBench.js",
-    "check:custom-bench": "npm run build && node ./test/bench/custom/main.mjs",
+    "check:granular-bench": "npm run build:bench && npm run check:baseline-bench && node ./test/bench/etc/run_granular_benchmarks.js",
+    "check:spec-bench": "npm run build:bench && npm run check:baseline-bench && node ./test/bench/lib/spec/bsonBench.js",
+    "check:custom-bench": "npm run build && npm run check:baseline-bench && node ./test/bench/custom/main.mjs",
+    "check:baseline-bench": "node ./test/bench/etc/cpuBaseline.js",
     "build:bench": "cd test/bench && npx tsc",
     "build:ts": "node ./node_modules/typescript/bin/tsc",
     "build:dts": "npm run build:ts && api-extractor run --typescript-compiler-folder node_modules/typescript --local && node etc/clean_definition_files.cjs",

package.json

@@ -1,22 +1,26 @@
 /* eslint-disable strict */
 import { BSON } from '../../../lib/bson.mjs';
 
-const ObjectId_isValid = [
-  function objectid_isvalid_wrong_string_length() {
-    BSON.ObjectId.isValid('a');
-  },
-  /** wrong character at the start, could be the most short circuited code path */
-  function objectid_isvalid_invalid_hex_at_start() {
-    BSON.ObjectId.isValid('g6e84ebdc96f4c0772f0cbbf');
-  },
-  /** wrong character at the end, could be the least short circuited code path */
-  function objectid_isvalid_invalid_hex_at_end() {
-    BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbg');
-  },
-  function objectid_isvalid_valid_hex_string() {
-    BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbf');
-  }
-];
+const ObjectId_isValid = {
+  name: 'ObjectId_isValid',
+  tags: ['alerting-benchmark', 'objectid'],
+  benchmarks: [
+    function objectid_isvalid_wrong_string_length() {
+      BSON.ObjectId.isValid('a');
+    },
+    /** wrong character at the start, could be the most short circuited code path */
+    function objectid_isvalid_invalid_hex_at_start() {
+      BSON.ObjectId.isValid('g6e84ebdc96f4c0772f0cbbf');
+    },
+    /** wrong character at the end, could be the least short circuited code path */
+    function objectid_isvalid_invalid_hex_at_end() {
+      BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbg');
+    },
+    function objectid_isvalid_valid_hex_string() {
+      BSON.ObjectId.isValid('66e84ebdc96f4c0772f0cbbf');
+    }
+  ]
+};
 
 // Add benchmarks here:
-export const benchmarks = [...ObjectId_isValid];
+export const suites = [ObjectId_isValid];

test/bench/custom/benchmarks.mjs

@@ -4,7 +4,7 @@ import util from 'node:util';
 import fs from 'node:fs';
 import os from 'node:os';
 import benchmark from 'benchmark';
-import { benchmarks } from './benchmarks.mjs';
+import { suites } from './benchmarks.mjs';
 
 const hw = os.cpus();
 const ram = os.totalmem() / 1024 ** 3;
@@ -20,20 +20,71 @@ const systemInfo = () =>
   ].join('\n');
 console.log(systemInfo());
 
-const suite = new benchmark.Suite();
-
-for (const bench of benchmarks) suite.add(bench.name, bench);
-
-suite
-  .on('cycle', function logBenchmark(event) {
-    console.log(String(event.target));
-  })
-  .on('complete', function outputPerfSend() {
-    const data = Array.from(this).map(bench => ({
-      info: { test_name: bench.name },
-      metrics: [{ name: 'ops_per_sec', value: bench.hz }]
-    }));
-    console.log(util.inspect(data, { depth: Infinity, colors: true }));
-    fs.writeFileSync('customBenchmarkResults.json', JSON.stringify(data), 'utf8');
-  })
-  .run();
+function logBenchmark(event) {
+  console.log(String(event.target));
+}
+
+function processBenchmarkResult(bench, metadata) {
+  return {
+    info: { test_name: bench.name },
+    metrics: [{ name: 'ops_per_sec', value: bench.hz, metadata }]
+  };
+}
+
+let completedSuites = 0;
+async function completeSuite() {
+  const metadata = { improvement_direction: 'up' };
+  if (++completedSuites >= collectedSuites.length) {
+    let cpuBaselineResults;
+    try {
+      cpuBaselineResults = await import('../etc/cpuBaseline.json', { assert: { type: 'json' } });
+    } catch (cause) {
+      throw new Error("Couldn't find baseline results", { cause });
+    }
+
+    cpuBaselineResults = cpuBaselineResults.default;
+    const cpuBaselineResult = cpuBaselineResults.hz;
+    if (typeof cpuBaselineResult !== 'number') {
+      throw new Error("Couldn't find baseline result");
+    }
+
+    const data = [];
+    for (const { suite, suiteConfig } of collectedSuites) {
+      const { tags } = suiteConfig;
+      for (const bench of Array.from(suite)) {
+        const result = processBenchmarkResult(bench, { ...metadata, tags });
+        result.metrics.push({
+          name: 'normalized_throughput',
+          value: bench.hz / cpuBaselineResult,
+          metadata: { ...metadata, tags }
+        });
+        data.push(result);
+      }
+
+      data.push({
+        info: { test_name: 'cpuBaseline' },
+        metrics: [{ name: 'ops_per_sec', value: cpuBaselineResult, metadata }]
+      });
+
+      console.log(util.inspect(data, { depth: Infinity, colors: true }));
+      fs.writeFileSync('customBenchmarkResults.json', JSON.stringify(data), 'utf8');
+    }
+  }
+}
+
+function processSuite(suiteModule, cycleHandler, completeHandler) {
+  let suite = new benchmark.Suite(suiteModule.name);
+  for (const b of suiteModule.benchmarks) {
+    suite.add(b.name, b);
+  }
+
+  suite = suite.on('cycle', cycleHandler).on('complete', completeHandler).run({ async: true });
+
+  return { suite, suiteConfig: suiteModule };
+}
+
+const collectedSuites = [];
+for (const suite of suites) {
+  const newSuite = processSuite(suite, logBenchmark, completeSuite);
+  collectedSuites.push(newSuite);
+}

test/bench/custom/main.mjs

@@ -4,21 +4,25 @@ In this directory are tests for code paths not covered by our spec or granular (
 
 ## How to write your own
 
-In `benchmarks.mjs` add a new test to an existing array or make a new array for a new subject area.
-Try to fit the name of the function into the format of: "subject area", "method or function" "test case that is being covered" (Ex. `objectid_isvalid_bestcase_false`).
+In `benchmarks.mjs` add a new test to an existing benchmark object or make a new object for a new subject area.
+Try to fit the name of the variables and the benchmark functions into the format of: "subject area", "method or function" "test case that is being covered" (Ex. `objectid_isvalid_bestcase_false`).
 Make sure your test is added to the `benchmarks` export.
 
 ### Example
 
 ```js
-const ObjectId_isValid = [
-  function objectid_isvalid_strlen() {
-    BSON.ObjectId.isValid('a');
-  },
-  // ...
-];
-
-export const benchmarks = [...ObjectId_isValid];
+const ObjectId_isValid = {
+  name: 'ObjectId_isValid',
+  tags: ['objectid'],
+  benchmarks : [
+    function objectid_isvalid_strlen() {
+      BSON.ObjectId.isValid('a');
+    },
+    // ...
+  ]
+};
+
+export const benchmarks = [ObjectId_isValid];
 ```
 
 ## Output
@@ -28,9 +32,9 @@ The JSON emitted at the end of the benchmarks must follow our performance tracki
 The JSON must be an array of "`Test`"s:
 
 ```ts
-type Metric = { name: string, value: number }
+type Metric = { name: string, value: number, metadata: { improvement_direction: 'up' | 'down' } }
 type Test = {
-    info: { test_name: string },
+    info: { test_name: string, tags?: string[]},
     metrics: Metric[]
 }
 ```

test/bench/custom/readme.md

@@ -1,2 +1,3 @@
 *Results.json
-resultsCollected.json
+cpuBaseline.json
+resultsCollected*.json