Run wasm tools in hub test instead of skipping them

hub/builders/emscripten.py

@@ -63,7 +63,12 @@ def build(tool_name, recipe_dir, emscripten_settings, source, output_dir="build"
         "-s FORCE_FILESYSTEM=1 "
         "-s EXPORTED_RUNTIME_METHODS=['callMain','FS','PROXYFS','WORKERFS'] "
         "-s MODULARIZE=1 "
-        "-s ENVIRONMENT=web,worker "
+        # node is included so that hub test can execute the artifact. Without
+        # it the module refuses to load outside a browser ("not compiled for
+        # this environment"), which is why wasm-only recipes could only ever be
+        # skipped. It costs about 3 KB of JS shim and changes nothing for the
+        # browser targets.
+        "-s ENVIRONMENT=web,worker,node "
         "-s ALLOW_MEMORY_GROWTH=1 "
         "-s EXIT_RUNTIME=1 "
         "-lworkerfs.js "

hub/hub.py

@@ -73,11 +73,13 @@ def test_cmd(args):
     failed_tests = test_tools(REGISTRY_DIR)
 
     if len(failed_tests) > 0:
+        # Exits non-zero so the failure reaches CI. Without this the result was
+        # printed and the command still succeeded, so a failing tool could not
+        # stop anything.
         print(f"The following tools failed the tests: {failed_tests}")
-    else:
-        print("All tests passed")
-
-    pass
+        raise SystemExit(1)
+
+    print("All tests passed")
 
 def sbom_cmd(args):
     #TODO

hub/tests/run_wasm.js

@@ -0,0 +1,136 @@
+// Runs one Emscripten-built tool under Node and reports what it produced.
+//
+// hub test can only exercise a tool it can execute, and most of the catalogue
+// has no native build at all -- those recipes were skipped, which meant a green
+// test run said only that the build had produced a file of the right name. This
+// runs the wasm artifact instead, so the same input generation and output type
+// checking apply to every recipe rather than to the few with a native binary.
+//
+// Reads a JSON spec from a file and writes a JSON result to stdout, so the
+// Python side does not have to know anything about Emscripten:
+//
+//   spec   { argv: [], stdin: "", files: { name: base64 } }
+//   result { loaded, completed, exitCode, stdout, stderr, files: { name: base64 } }
+//
+// loaded=false means the module would not start at all -- most often because it
+// was built without "node" in -sENVIRONMENT. completed=false means it started
+// and then trapped. Neither is expressible as an exit status, because a tool
+// returning -1 to reject its arguments is ordinary.
+//
+// Every regular file left in the working directory is returned, not a named
+// list: the caller does not tell the tool where to write, it looks afterwards
+// for a file named after the output, so the same discovery has to work here.
+//
+// Usage: node run_wasm.js <module.js> <spec.json>
+
+const fs = require("fs");
+const path = require("path");
+
+async function main() {
+  const [, , modulePath, specPath] = process.argv;
+  if (!modulePath || !specPath) {
+    throw new Error("usage: run_wasm.js <module.js> <spec.json>");
+  }
+
+  const spec = JSON.parse(fs.readFileSync(specPath, "utf8"));
+  const resolved = path.resolve(modulePath);
+  const factory = require(resolved);
+
+  // Handed over rather than left for the module to fetch. A module built for
+  // the browser resolves its .wasm with fetch or XHR, neither of which exists
+  // here.
+  const wasmBinary = fs.readFileSync(resolved.replace(/\.js$/, ".wasm"));
+
+  let stdout = "";
+  let stderr = "";
+
+  // The tool reads stdin a byte at a time and expects null at the end.
+  const stdinBytes = Buffer.from(spec.stdin ?? "", "utf8");
+  let stdinPos = 0;
+
+  let Module;
+  try {
+    Module = await factory({
+      wasmBinary,
+    // The recipes are built with INVOKE_RUN=0, so main is called explicitly
+    // below once the input files are in place.
+    noInitialRun: true,
+    print: (line) => {
+      stdout += line + "\n";
+    },
+    printErr: (line) => {
+      stderr += line + "\n";
+    },
+      stdin: () => (stdinPos < stdinBytes.length ? stdinBytes[stdinPos++] : null),
+    });
+  } catch (err) {
+    // A module built without "node" in -sENVIRONMENT refuses to start here. That
+    // says nothing about whether the tool works, so it is reported as its own
+    // outcome rather than as a failure -- the caller decides what to do with it.
+    const message = String((err && err.message) || err);
+    const unsupported = /not compiled for this environment|not enabled at build time/.test(message);
+    process.stdout.write(JSON.stringify({
+      loaded: false,
+      unsupportedEnvironment: unsupported,
+      stdout: "",
+      stderr: message,
+      files: {},
+    }));
+    return;
+  }
+
+  for (const [name, contents] of Object.entries(spec.files ?? {})) {
+    Module.FS.writeFile(name, Buffer.from(contents, "base64"));
+  }
+
+  let exitCode = 0;
+  let completed = true;
+  try {
+    // callMain returns main's value. It does not throw for a normal return,
+    // even with EXIT_RUNTIME=1 -- reading the status from a thrown ExitStatus
+    // alone would report every tool as having exited 0.
+    const status = Module.callMain(spec.argv ?? []);
+    if (typeof status === "number") exitCode = status;
+  } catch (err) {
+    if (err && err.name === "ExitStatus") {
+      // A program that calls exit() rather than returning does throw.
+      exitCode = err.status;
+    } else {
+      // Reported in its own field. Overloading a status value cannot work:
+      // "return -1" is an ordinary way for a tool to reject its arguments, and
+      // would be indistinguishable from a trap.
+      completed = false;
+      stderr += String((err && err.message) || err) + "\n";
+    }
+  }
+
+  // Everything the tool left behind, so the caller can look for outputs the
+  // same way it does for a native binary. The virtual root also holds the
+  // mount points Emscripten sets up, hence the regular-file check.
+  const files = {};
+  for (const name of Module.FS.readdir("/")) {
+    if (name === "." || name === "..") continue;
+    try {
+      const stat = Module.FS.stat("/" + name);
+      if (!Module.FS.isFile(stat.mode)) continue;
+      files[name] = Buffer.from(Module.FS.readFile("/" + name)).toString("base64");
+    } catch {
+      // Unreadable entries are simply not reported.
+    }
+  }
+
+  process.stdout.write(JSON.stringify({ loaded: true, completed, exitCode, stdout, stderr, files }));
+}
+
+main().catch((err) => {
+  // Reported in the same shape as a normal result so the caller has one path.
+  process.stdout.write(
+    JSON.stringify({
+      loaded: false,
+      unsupportedEnvironment: false,
+      stdout: "",
+      stderr: String((err && err.stack) || err),
+      files: {},
+    })
+  );
+});

hub/tests/test.py

@@ -1,6 +1,7 @@
 import os
 import subprocess
 import json
+import base64
 import shutil
 import tempfile
 from pathlib import Path
@@ -41,14 +42,113 @@ def test_tools(registry_dir):
             with open(bundle_path) as f:
                 tool_bundle = json.load(f)
 
-            if not test_tool_outputs(version_dir, tool_bundle):
+            # Guarded so that one tool cannot end the run. Anything unexpected
+            # here previously escaped as a traceback and left every remaining
+            # recipe untested, which is the opposite of what this command is for.
+            try:
+                ok = test_tool_outputs(version_dir, tool_bundle)
+            except Exception as err:
+                print(f"[Error] Testing {tool_bundle.get('name')} raised {type(err).__name__}: {err}")
+                ok = False
+
+            if not ok:
                 failed.append(tool_bundle["name"])
 
     return failed
 
 
 example_inputs = get_example_inputs()
 
+WASM_HARNESS = Path(__file__).resolve().parent / "run_wasm.js"
+
+
+def safe(text):
+    """Makes text printable.
+
+    A tool's output reaches us as a JSON string decoded from whatever bytes it
+    wrote, so it can contain lone surrogates. print() cannot encode those, and
+    the resulting UnicodeEncodeError escaped test_tool_outputs and left every
+    remaining recipe untested.
+    """
+    return (text or "").encode("utf-8", errors="replace").decode("utf-8")
+
+
+def run_wasm_tool(wasm_path, argv, tool_input, input_files, tmp_path):
+    """Runs a wasm tool through the Node harness.
+
+    Returns (stdout, stderr), or None if the tool could not be run at all --
+    which is a test failure rather than something to report as tool output.
+    Files the tool produced are written into tmp_path, so the caller inspects
+    the results exactly as it does for a native binary.
+    """
+    spec = {
+        "argv": argv,
+        "stdin": tool_input.strip() if tool_input else "",
+        "files": {
+            name: base64.b64encode((tmp_path / name).read_bytes()).decode("ascii")
+            for name in input_files
+        },
+    }
+
+    spec_path = tmp_path / "_wasm_spec.json"
+    spec_path.write_text(json.dumps(spec))
+
+    try:
+        result = subprocess.run(
+            ["node", str(WASM_HARNESS), str(wasm_path), str(spec_path)],
+            capture_output=True,
+            timeout=60,
+        )
+    except subprocess.TimeoutExpired:
+        print("[Error] Tool execution timed out")
+        return None
+    except FileNotFoundError:
+        print("[Error] node is required to test wasm tools and was not found")
+        return None
+
+    try:
+        run = json.loads(result.stdout.decode("utf-8", errors="replace"))
+    except json.JSONDecodeError:
+        print("[Error] The wasm harness produced no usable result")
+        print(result.stdout.decode("utf-8", errors="replace")[:500])
+        print(result.stderr.decode("utf-8", errors="replace")[:500])
+        return None
+
+    for name, contents in run.get("files", {}).items():
+        # Inputs are written back too. A tool editing its input in place is a
+        # normal pattern, and under the native runtime it writes straight into
+        # this directory, so skipping them here would make the two runtimes
+        # disagree about what the tool produced.
+        if name == spec_path.name:
+            continue
+        (tmp_path / name).write_bytes(base64.b64decode(contents))
+
+    if not run.get("loaded"):
+        if run.get("unsupportedEnvironment"):
+            # Built without "node" in -sENVIRONMENT, so it refuses to start
+            # outside a browser. That is a property of how it was compiled, not
+            # evidence that the tool is broken, so it is skipped rather than
+            # failed -- reporting it as a failure would turn every recipe built
+            # this way red for a reason unrelated to the recipe.
+            print("[SKIP] Built without node support, so it cannot be run here")
+            return "", ""
+
+        print("[Error] The wasm module failed to load")
+        print(safe(run.get("stderr"))[:600])
+        return None
+
+    if not run.get("completed"):
+        print("[Error] The wasm module trapped part way through")
+        print(safe(run.get("stderr"))[:600])
+        return None
+
+    # A tool's own non-zero status is only reported, matching how the native
+    # path treats it, since some tools exit non-zero by design.
+    if run.get("exitCode"):
+        print(f"[WARNING] Tool exited with status {run['exitCode']}")
+
+    return run.get("stdout", ""), run.get("stderr", "")
+
 def test_tool_outputs(tool_dir, tool_bundle):
     base_dir = os.getcwd()
 
@@ -68,13 +168,25 @@ def test_tool_outputs(tool_dir, tool_bundle):
 
             bin_name = tool_bundle.get("bin")
             bin_path = tmp_path / "runtime" / "native" / bin_name
-
-            if not bin_path.is_file():
-                print(f"[SKIP] Binary not found: {bin_name}")
+            wasm_path = tmp_path / "runtime" / "wasm" / f"{bin_name}.js"
+
+            # Most of the catalogue has no native build, and those recipes used
+            # to be skipped -- so a green run said only that the build had
+            # produced a file of the right name, and anything that linked but
+            # trapped at run time went unnoticed. Fall back to the wasm artifact,
+            # which every recipe has.
+            runtime = None
+            if bin_path.is_file():
+                runtime = "native"
+            elif wasm_path.is_file():
+                runtime = "wasm"
+            else:
+                print(f"[SKIP] No native or wasm artifact found for: {bin_name}")
                 return True
 
             tool_input = ""
-            cmd = [str(bin_path)]
+            input_files = []
+            cmd = [str(bin_path)] if runtime == "native" else []
 
             # Parameters
             for parameter in tool_bundle.get("parameters", []):
@@ -108,30 +220,40 @@ def test_tool_outputs(tool_dir, tool_bundle):
                     with open(file_path, "w") as f:
                         f.write(example_inputs[input_type])
 
+                    input_files.append(file_name)
+
                     if input_def.get("flag"):
                         cmd.append(input_def["flag"])
 
-                    cmd.append(str(file_path))
+                    # A wasm tool sees its own virtual filesystem, where the
+                    # host path means nothing.
+                    cmd.append(file_name if runtime == "wasm" else str(file_path))
 
                 else:
                     print(f"[TODO] Unsupported input mode: {input_def}")
                     return False
 
             # Run tool
-            print(f"Testing tool {tool_bundle['name']} with command {cmd}")
-            try:
-                result = subprocess.run(
-                    cmd,
-                    input=tool_input.strip().encode("ascii") if tool_input else None,
-                    capture_output=True,
-                    timeout=10,
-                )
-            except subprocess.TimeoutExpired:
-                print("[Error] Tool execution timed out")
-                return False
-
-            stdout = result.stdout.decode("ascii", errors="replace")
-            stderr = result.stderr.decode("ascii", errors="replace")
+            print(f"Testing tool {tool_bundle['name']} ({runtime}) with command {cmd}")
+            if runtime == "wasm":
+                run = run_wasm_tool(wasm_path, cmd, tool_input, input_files, tmp_path)
+                if run is None:
+                    return False
+                stdout, stderr = run
+            else:
+                try:
+                    result = subprocess.run(
+                        cmd,
+                        input=tool_input.strip().encode("ascii") if tool_input else None,
+                        capture_output=True,
+                        timeout=10,
+                    )
+                except subprocess.TimeoutExpired:
+                    print("[Error] Tool execution timed out")
+                    return False
+
+                stdout = result.stdout.decode("ascii", errors="replace")
+                stderr = result.stderr.decode("ascii", errors="replace")
 
             all_ok = True
 
@@ -165,7 +287,7 @@ def test_tool_outputs(tool_dir, tool_bundle):
                 if not detected:
                     print(f"[WARNING] Empty output ({output_name}, {tool_bundle['name']}, {cmd})")
                     print("stderr:")
-                    print(stderr.strip())
+                    print(safe(stderr).strip())
                 elif detected not in output_def["types"]:
                     print(f"[ERROR] Unexpected output type ({output_name}, {tool_bundle['name']}, {cmd})")
                     print(f"  Detected : {detected}")