[CI] update image diff framework and update to node 20 (#13944)

* update image diff framework

* upload diffs

* change to 0.1 per pixel

* add simple reporter

* needed changes to package

* update lock

* update package lock

* force greenyellow as background image for transparency

* remove commented code

jest.config.ts

@@ -79,6 +79,6 @@ const createProject = (type: string) => {
 // Sync object
 const config: Config.InitialOptions = {
     projects: [createProject("unit"), createProject("visualization"), createProject("integration"), createProject("performance"), createProject("interactions")],
-    reporters: ["default", "jest-screenshot/reporter", "jest-junit"],
+    reporters: ["default", "./scripts/jest-imagediff-reporter", "jest-junit"],
 };
 export default config;

jest.visualization.setup.afterEnv.ts

@@ -1,3 +1,5 @@
-import { setupJestScreenshot } from "jest-screenshot";
-setupJestScreenshot();
+const { configureToMatchImageSnapshot } = require("jest-image-snapshot");
+
+const toMatchImageSnapshot = configureToMatchImageSnapshot({});
+expect.extend({ toMatchImageSnapshot });
 jest.setTimeout(12000000);

package.json

@@ -30,7 +30,7 @@
         "eslint-formatter-azure-devops": "^0.2.0",
         "eslint-plugin-import": "~2.26.0",
         "eslint-plugin-jest": "~26.5.3",
-        "eslint-plugin-jsdoc": "~39.3.2",
+        "eslint-plugin-jsdoc": "~46.2.6",
         "eslint-plugin-prettier": "~4.0.0",
         "eslint-plugin-tsdoc": "~0.2.14",
         "fs-extra": "^10.0.1",
@@ -95,7 +95,7 @@
         "start": "concurrently --kill-others -m 2 -n dev-watch,cdn-server \"npm run watch:dev:skip-compile\" \"npm run serve -w @tools/babylon-server\""
     },
     "engines": {
-        "node": ">=14.0.0 <20.0.0",
+        "node": ">=14.0.0 <21.0.0",
         "npm": ">=8.0.0"
     },
     "changelog": {

packages/tools/babylonServer/public/empty.html

@@ -24,6 +24,7 @@
             canvas {
                 width: 100%;
                 height: 100%;
+                background: greenyellow !important;
             }
 
             #inspector {

packages/tools/tests/package.json

@@ -31,12 +31,12 @@
         "jest": "^27.4.7",
         "jest-puppeteer": "^6.0.3",
         "sass": "^1.62.1",
-        "puppeteer": "^13.1.2",
+        "puppeteer": "^20.5.0",
         "rimraf": "^3.0.2",
         "typescript": "^4.4.4"
     },
     "optionalDependencies": {
-        "jest-screenshot": "^0.3.5"
+        "jest-image-snapshot": "^6.1.0"
     },
     "sideEffects": false
 }

packages/tools/tests/test/visualization/visualization.utils.ts

@@ -1,4 +1,3 @@
-import * as jestScreenshot from "jest-screenshot";
 import * as path from "path";
 import * as fs from "fs";
 import {
@@ -126,9 +125,7 @@ export const evaluateTests = async (engineType = "webgl2", testFileName = "confi
     // });
 
     test /*.concurrent*/
-        .each(
-            tests
-        )(
+        .each(tests)(
         "$title",
         async (test) => {
             log(`Running - ${test.title}`);
@@ -149,15 +146,18 @@ export const evaluateTests = async (engineType = "webgl2", testFileName = "confi
                 // Take screenshot
                 const screenshot = await page.screenshot();
 
-                jestScreenshot.setupJestScreenshot({
-                    pixelThresholdRelative: (test.errorRatio || 2.5) / 100,
-                });
+                const directory = path.resolve(__dirname, "../../../../../jest-screenshot-report");
+
                 // Test screenshot (also save this new screenshot if -u is set)
                 expect(screenshot).toMatchImageSnapshot({
-                    path: path.resolve(
-                        __dirname,
-                        `./ReferenceImages/${useStandardTestList ? "" : testFileName}/${test.referenceImage ? test.referenceImage : test.title + ".png"}`
-                    ),
+                    customDiffConfig: {
+                        threshold: 0.1,
+                    },
+                    customSnapshotsDir: path.resolve(__dirname, `./ReferenceImages/${useStandardTestList ? "" : testFileName}/`),
+                    customSnapshotIdentifier: (test.referenceImage ? test.referenceImage : test.title).replace(".png", ""),
+                    failureThreshold: (test.errorRatio || 2.5) / 100,
+                    failureThresholdType: "percent",
+                    customDiffDir: directory,
                 });
             } finally {
                 // dispose the scene

scripts/jest-imagediff-reporter.js

@@ -0,0 +1,50 @@
+const fs = require("fs");
+const path = require("path");
+
+const DIFF_DIRECTORY = path.join(__dirname, "..", "jest-screenshot-report");
+
+class ImageDiffReporter {
+    /* istanbul ignore next - test coverage in child process */
+    onRunStart() {
+        try {
+            fs.statSync(DIFF_DIRECTORY);
+        } catch (e) {
+            fs.mkdirSync(DIFF_DIRECTORY, { recursive: true });
+        }
+
+        // cleanup
+        const files = fs.readdirSync(DIFF_DIRECTORY);
+
+        if (files.length > 0) {
+            files.forEach(function (filename) {
+                if (fs.statSync(DIFF_DIRECTORY + "/" + filename).isDirectory()) {
+                    this.removeDir(DIFF_DIRECTORY + "/" + filename);
+                } else {
+                    fs.unlinkSync(DIFF_DIRECTORY + "/" + filename);
+                }
+            });
+        }
+    }
+
+    /* istanbul ignore next - test coverage in child process */
+    onRunComplete() {
+        const files = fs.readdirSync(DIFF_DIRECTORY);
+        const imagesHtml = files.map((file) => `<h3>${file.replace("-diff.png", "")}</h3><img src="./${file}" />`).join("\n");
+        // write index.html
+        const html = `
+
+            <html>
+                <head>
+                    <title>Diff reporter</title>
+                    </head>
+                    <body>
+                        <h1>Diff reporter</h1>
+                        ${imagesHtml}
+                    </body>
+                </html>
+            `;
+        fs.writeFileSync(path.join(DIFF_DIRECTORY, "index.html"), html);
+    }
+}
+
+module.exports = ImageDiffReporter;