Revamp file-based data access

- New datasource URL syntax based on ZEP 8
proposal (zarr-developers/zeps#48)
- Support for ZIP archives

Author: jbms

.github/workflows/build.yml

@@ -31,11 +31,16 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-node@v4
         with:
-          node-version: 20.x
+          node-version: 22.x
           cache: "npm"
           cache-dependency-path: |
             package-lock.json
             examples/**/package-lock.json
+      # go needed for fake_gcs_server used by the javascript tests
+      - name: Setup go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "stable"
       - run: npm install
       - run: npm run format:fix
       - name: Check for dirty working directory
@@ -99,7 +104,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-node@v4
         with:
-          node-version: 20.x
+          node-version: 22.x
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
@@ -146,7 +151,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-node@v4
         with:
-          node-version: 20.x
+          node-version: 22.x
           cache: "npm"
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -221,7 +226,7 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: 20.x
+          node-version: 22.x
           registry-url: "https://registry.npmjs.org"
       - uses: actions/download-artifact@v4
         with:

.github/workflows/build_preview.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       matrix:
         node-version:
-          - "20.x"
+          - "22.x"
     runs-on: ubuntu-latest
 
     steps:

.prettierignore

@@ -1,9 +1,12 @@
 /templates/
 /python/
 /third_party/jpgjs/jpg.js
-/testdata/*.json
+/testdata/**/*.json
+zarr.json
 .parcel-cache
 dist
 /lib
 /docs/_build/
 /.ruff_cache
+package.json
+package-lock.json

build_tools/build-package.ts

@@ -71,6 +71,7 @@ async function buildPackage(options: {
     outbase: srcDir,
     bundle: false,
     outdir: libDir,
+    target: "es2022",
   });
 
   let compilerOptionsFromConfigFile: ts.CompilerOptions = {};

build_tools/update-conditions.ts

@@ -17,70 +17,100 @@ const imports: Record<string, any> = {};
 imports["#src/third_party/jpgjs/jpg.js"] = "./src/third_party/jpgjs/jpg.js";
 imports["#src/*.js"] = "./src/*.ts";
 imports["#src/*"] = "./src/*";
+imports["#tests/fixtures/msw"] = {
+  node: "./tests/fixtures/msw_node.ts",
+  default: "./tests/fixtures/msw_browser.ts",
+};
+imports["#tests/fixtures/gl"] = {
+  node: "./tests/fixtures/gl_node.ts",
+  default: "./tests/fixtures/gl_browser.ts",
+};
+imports["#tests/*.js"] = "./tests/*.ts";
 imports["#testdata/*"] = "./testdata/*";
 
-const datasourceDir = path.resolve(rootDir, "src", "datasource");
-const layerDir = path.resolve(rootDir, "src", "layer");
-
-const datasources = (
-  await fs.promises.readdir(datasourceDir, { withFileTypes: true })
-)
-  .filter((e) => e.isDirectory())
-  .map((e) => e.name);
-
-const layers = (await fs.promises.readdir(layerDir, { withFileTypes: true }))
-  .filter((e) => e.isDirectory())
-  .map((e) => e.name);
-
-const datasourceKeys = {
-  backend: "backend",
-  async_computation: "async_computation",
-  register_default: "frontend",
-  register_credentials_provider: "frontend",
-} as const;
+async function listSubdirs(dir: string): Promise<string[]> {
+  return (await fs.promises.readdir(dir, { withFileTypes: true }))
+    .filter((e) => e.isDirectory())
+    .map((e) => e.name);
+}
 
-const datasourceModules = Object.fromEntries(
-  Object.values(datasourceKeys).map((key) => [key, new Array<string>()]),
-);
+async function writeModule(modulePath: string, imports: string[]) {
+  await fs.promises.writeFile(
+    modulePath,
+    "// DO NOT EDIT: Generated by config/update_conditions.ts\n" +
+      imports.map((name) => `import ${JSON.stringify(name)};\n`).join(""),
+    { encoding: "utf-8" },
+  );
+}
 
-for (const datasource of datasources) {
-  for (const [filePrefix, moduleKind] of Object.entries(datasourceKeys)) {
-    const sourcePrefix = `./src/datasource/${datasource}/${filePrefix}`;
-    if (
-      await fs.promises
-        .stat(path.resolve(rootDir, `${sourcePrefix}.ts`))
-        .catch(() => undefined)
-    ) {
-      const source = sourcePrefix + JS_EXT;
-      const conditions: Record<string, string> = {};
-      if (datasource === "python") {
-        conditions["neuroglancer/python"] = source;
-        conditions.default = NOOP;
-      } else {
-        if (filePrefix === "register_credentials_provider") {
-          conditions["neuroglancer/python"] = NOOP;
+async function handleDrivers(
+  kind: string,
+  moduleMap: Record<string, string[]>,
+) {
+  const driverDir = path.resolve(rootDir, "src", kind);
+  const drivers = await listSubdirs(driverDir);
+  const modules: Record<string, string[]> = {};
+  for (const driver of drivers) {
+    for (const [filePrefix, moduleKinds] of Object.entries(moduleMap)) {
+      const sourcePrefix = `./src/${kind}/${driver}/${filePrefix}`;
+      if (
+        await fs.promises
+          .stat(path.resolve(rootDir, `${sourcePrefix}.ts`))
+          .catch(() => undefined)
+      ) {
+        const source = sourcePrefix + JS_EXT;
+        const conditions: Record<string, string> = {};
+        if (driver === "python") {
+          conditions["neuroglancer/python"] = source;
+          conditions.default = NOOP;
+        } else {
+          if (filePrefix === "register_credentials_provider") {
+            conditions["neuroglancer/python"] = NOOP;
+          }
+          conditions[`neuroglancer/${kind}/${driver}:enabled`] = source;
+          conditions[`neuroglancer/${kind}:none_by_default`] = NOOP;
+          conditions[`neuroglancer/${kind}/${driver}:disabled`] = NOOP;
+          conditions.default = source;
+        }
+        let moduleId = `#${kind}/${driver}`;
+        if (filePrefix !== "index") {
+          moduleId += `/${filePrefix}`;
+        }
+        imports[moduleId] = conditions;
+        for (const moduleKind of moduleKinds) {
+          if (modules[moduleKind] === undefined) {
+            modules[moduleKind] = [];
+          }
+          modules[moduleKind].push(moduleId);
         }
-        conditions[`neuroglancer/datasource/${datasource}:enabled`] = source;
-        conditions["neuroglancer/datasource:none_by_default"] = NOOP;
-        conditions[`neuroglancer/datasource/${datasource}:disabled`] = source;
-        conditions.default = source;
       }
-      const moduleId = `#datasource/${datasource}/${filePrefix}`;
-      imports[moduleId] = conditions;
-      datasourceModules[moduleKind].push(moduleId);
     }
   }
+  for (const [moduleKind, moduleIds] of Object.entries(modules)) {
+    await writeModule(
+      path.resolve(driverDir, `enabled_${moduleKind}_modules.ts`),
+      moduleIds,
+    );
+  }
 }
 
-for (const layer of layers) {
-  const source = `./src/layer/${layer}/index` + JS_EXT;
-  imports[`#layer/${layer}`] = {
-    [`neuroglancer/layer/${layer}:enabled`]: source,
-    "neuroglancer/layer:none_by_default": NOOP,
-    [`neuroglancer/layer/${layer}:enabled`]: source,
-    default: source,
-  };
-}
+await handleDrivers("datasource", {
+  backend: ["backend"],
+  async_computation: ["async_computation"],
+  register_default: ["frontend"],
+  register_credentials_provider: ["frontend"],
+});
+
+await handleDrivers("kvstore", {
+  register: ["frontend", "backend"],
+  register_frontend: ["frontend"],
+  register_backend: ["backend"],
+  register_credentials_provider: ["frontend"],
+});
+
+await handleDrivers("layer", {
+  index: ["frontend"],
+});
 
 // main entrypoint.
 imports["#main"] = {
@@ -94,32 +124,6 @@ imports["#python_integration_build"] = {
   default: NOOP,
 };
 
-async function writeModule(modulePath: string, imports: string[]) {
-  await fs.promises.writeFile(
-    modulePath,
-    "// DO NOT EDIT: Generated by config/update_conditions.ts\n" +
-      imports.map((name) => `import ${JSON.stringify(name)};\n`).join(""),
-    { encoding: "utf-8" },
-  );
-}
-
-for (const [moduleKind, moduleIds] of Object.entries(datasourceModules)) {
-  await writeModule(
-    path.resolve(
-      rootDir,
-      "src",
-      "datasource",
-      `enabled_${moduleKind}_modules.ts`,
-    ),
-    moduleIds,
-  );
-}
-
-await writeModule(
-  path.resolve(rootDir, "src", "layer", "enabled_frontend_modules.ts"),
-  layers.map((name) => `#layer/${name}`),
-);
-
 packageJson.imports = imports;
 
 packageJson.exports = {

build_tools/vitest/build_fake_gcs_server.ts

@@ -0,0 +1,57 @@
+/**
+ * @license
+ * Copyright 2024 Google Inc.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { spawnSync } from "node:child_process";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+export async function getFakeGcsServerBin(): Promise<string> {
+  const binDir = path.join(
+    import.meta.dirname,
+    "..",
+    "..",
+    "node_modules",
+    ".cache",
+    "gobin",
+  );
+  const serverBinPath =
+    path.join(binDir, "fake-gcs-server") +
+    (process.platform === "win32" ? ".exe" : "");
+  if (
+    !(await fs.access(serverBinPath).then(
+      () => true,
+      () => false,
+    ))
+  ) {
+    console.log("Building fake-gcs-server");
+    // Note: For unknown reasons, using `await promisify(spawn)` in place of
+    // `spawnSync` causes the vitest process to exit as soon as the child
+    // process completes.
+    spawnSync(
+      "go",
+      [
+        "install",
+        "github.com/fsouza/fake-gcs-server@3b3d059cbaade55b480196a51dedb7aa82ec2b0a",
+      ],
+      {
+        env: { ...process.env, GOBIN: binDir },
+        stdio: ["ignore", "inherit", "inherit"],
+      },
+    );
+    console.log("Done building fake-gcs-server");
+  }
+  return serverBinPath;
+}