windows support (#944)

* disable autocrlf

* test on windows

* cross-spawn

* rimraf

* use cross-env to run mocha

* fix tests with posix paths

* one less op

* mostly posix

Co-authored-by: Mike Bostock <[email protected]>

* fix data loader and config tests

* more tests

* all tests passing

* tweak gitattributes

* test fixes

* .gitattributes, take 3?

* fix empty inversion

* fix isEmpty logic, again

* Update src/files.ts

* add rimraf to templates

---------

Co-authored-by: Mike Bostock <[email protected]>
Co-authored-by: Mike Bostock <[email protected]>

Author: mythmon

.gitattributes

@@ -0,0 +1,2 @@
+# Ask git to not change line endings (to preserve content hashes).
+* -text

.github/workflows/test.yml

@@ -11,7 +11,9 @@ jobs:
     strategy:
       matrix:
         version: [20, 21]
-    runs-on: ubuntu-latest
+        os: [ubuntu-latest, windows-latest]
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -28,7 +30,7 @@ jobs:
       - uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: test-output-changes
+          name: test-output-changes-${{ matrix.os }}-${{ matrix.version }}
           path: |
             test/output/*-changed.*
             test/output/build/*-changed/

bin/observable-init.js

@@ -1,4 +1,15 @@
-#!/usr/bin/env -S node --no-warnings=ExperimentalWarning
+#!/usr/bin/env node
+import {fileURLToPath} from "node:url";
+import crossSpawn from "cross-spawn";
 
-await import("tsx/esm");
-await import("./observable.ts");
+crossSpawn.sync(
+  "node",
+  [
+    "--no-warnings=ExperimentalWarning",
+    "--import",
+    "tsx/esm",
+    fileURLToPath(import.meta.resolve("./observable.ts")),
+    ...process.argv.slice(2)
+  ],
+  {stdio: "inherit"}
+);

package.json

@@ -22,13 +22,13 @@
     "observable": "bin/observable-init.js"
   },
   "scripts": {
-    "dev": "rm -f docs/themes.md docs/theme/*.md && (tsx watch docs/theme/generate-themes.ts & tsx watch --no-warnings=ExperimentalWarning ./bin/observable.ts preview --no-open)",
-    "build": "yarn rebuild-themes && rm -rf dist && tsx --no-warnings=ExperimentalWarning ./bin/observable.ts build",
+    "dev": "rimraf --glob docs/themes.md docs/theme/*.md && (tsx watch docs/theme/generate-themes.ts & tsx watch --no-warnings=ExperimentalWarning ./bin/observable.ts preview --no-open)",
+    "build": "yarn rebuild-themes && rimraf dist && tsx --no-warnings=ExperimentalWarning ./bin/observable.ts build",
     "deploy": "yarn rebuild-themes && tsx --no-warnings=ExperimentalWarning ./bin/observable.ts deploy",
-    "rebuild-themes": "rm -f docs/themes.md docs/theme/*.md && tsx docs/theme/generate-themes.ts",
+    "rebuild-themes": "rimraf --glob docs/themes.md docs/theme/*.md && tsx docs/theme/generate-themes.ts",
     "test": "yarn test:mocha && yarn test:tsc && yarn test:lint && yarn test:prettier",
     "test:coverage": "c8 yarn test:mocha",
-    "test:mocha": "rm -rf test/.observablehq/cache test/input/build/*/.observablehq/cache && OBSERVABLE_TELEMETRY_DISABLE=1 TZ=America/Los_Angeles tsx --no-warnings=ExperimentalWarning ./node_modules/.bin/mocha 'test/**/*-test.*'",
+    "test:mocha": "rimraf --glob test/.observablehq/cache test/input/build/*/.observablehq/cache && cross-env OBSERVABLE_TELEMETRY_DISABLE=1 TZ=America/Los_Angeles tsx --no-warnings=ExperimentalWarning ./node_modules/mocha/bin/mocha.js 'test/**/*-test.*'",
     "test:lint": "eslint src test --max-warnings=0",
     "test:prettier": "prettier --check src test",
     "test:tsc": "tsc --noEmit",
@@ -53,6 +53,8 @@
     "acorn": "^8.11.2",
     "acorn-walk": "^8.3.0",
     "ci-info": "^4.0.0",
+    "cross-env": "^7.0.3",
+    "cross-spawn": "^7.0.3",
     "esbuild": "^0.19.8",
     "fast-array-diff": "^1.1.0",
     "gray-matter": "^4.0.3",
@@ -78,6 +80,7 @@
     "ws": "^8.14.2"
   },
   "devDependencies": {
+    "@types/cross-spawn": "^6.0.6",
     "@types/d3-array": "^3.2.1",
     "@types/he": "^1.2.3",
     "@types/jsdom": "^21.1.6",
@@ -103,6 +106,7 @@
     "fast-deep-equal": "^3.1.3",
     "mocha": "^10.2.0",
     "prettier": "^3.0.3 <3.1",
+    "rimraf": "^5.0.5",
     "typescript": "^5.2.2",
     "undici": "^5.27.2"
   },

src/build.ts

@@ -1,7 +1,7 @@
 import {createHash} from "node:crypto";
 import {existsSync} from "node:fs";
 import {access, constants, copyFile, readFile, writeFile} from "node:fs/promises";
-import {basename, dirname, extname, join} from "node:path";
+import {basename, dirname, extname, join} from "node:path/posix";
 import type {Config} from "./config.js";
 import {Loader} from "./dataloader.js";
 import {CliError, isEnoent} from "./error.js";

src/config.ts

@@ -1,4 +1,7 @@
-import {basename, dirname, join} from "node:path";
+import op from "node:path";
+import {basename, dirname, join} from "node:path/posix";
+import {cwd} from "node:process";
+import {pathToFileURL} from "node:url";
 import {visitMarkdownFiles} from "./files.js";
 import {formatIsoDate, formatLocaleDate} from "./format.js";
 import {parseMarkdown} from "./markdown.js";
@@ -51,7 +54,7 @@ export interface Config {
 
 export async function readConfig(configPath?: string, root?: string): Promise<Config> {
   if (configPath === undefined) return readDefaultConfig(root);
-  const importPath = join(process.cwd(), root ?? ".", configPath);
+  const importPath = pathToFileURL(op.join(cwd(), root ?? ".", configPath)).toString();
   return normalizeConfig((await import(importPath)).default, root);
 }
 

src/convert.ts

@@ -1,6 +1,6 @@
 import {existsSync} from "node:fs";
 import {utimes, writeFile} from "node:fs/promises";
-import {join} from "node:path";
+import {join} from "node:path/posix";
 import * as clack from "@clack/prompts";
 import wrapAnsi from "wrap-ansi";
 import type {ClackEffects} from "./clack.js";

src/create.ts

@@ -1,7 +1,8 @@
 import {exec} from "node:child_process";
 import {accessSync, existsSync, readdirSync, statSync} from "node:fs";
 import {constants, copyFile, mkdir, readFile, readdir, stat, writeFile} from "node:fs/promises";
-import {basename, dirname, join, normalize, resolve} from "node:path";
+import op from "node:path";
+import {basename, dirname, join, normalize} from "node:path/posix";
 import {setTimeout as sleep} from "node:timers/promises";
 import {fileURLToPath} from "node:url";
 import {promisify} from "node:util";
@@ -91,7 +92,7 @@ export async function create(options = {}, effects: CreateEffects = defaultEffec
         const s = clack.spinner();
         s.start("Copying template files");
         const template = includeSampleFiles ? "default" : "empty";
-        const templateDir = resolve(fileURLToPath(import.meta.url), "..", "..", "templates", template);
+        const templateDir = op.resolve(fileURLToPath(import.meta.url), "..", "..", "templates", template);
         const runCommand = packageManager === "yarn" ? "yarn" : `${packageManager ?? "npm"} run`;
         const installCommand = `${packageManager ?? "npm"} install`;
         await effects.sleep(1000);

src/dataloader.ts

@@ -1,8 +1,8 @@
-import {spawn} from "node:child_process";
 import {type WriteStream, createReadStream, existsSync, statSync} from "node:fs";
 import {mkdir, open, readFile, rename, unlink} from "node:fs/promises";
-import {dirname, extname, join} from "node:path";
+import {dirname, extname, join} from "node:path/posix";
 import {createGunzip} from "node:zlib";
+import {spawn} from "cross-spawn";
 import JSZip from "jszip";
 import {extract} from "tar-stream";
 import {maybeStat, prepareOutput} from "./files.js";

src/deploy.ts

@@ -1,4 +1,4 @@
-import {join} from "node:path";
+import {join} from "node:path/posix";
 import * as clack from "@clack/prompts";
 import wrapAnsi from "wrap-ansi";
 import type {BuildEffects} from "./build.js";

src/fileWatchers.ts

@@ -1,5 +1,5 @@
 import {type FSWatcher, existsSync, watch} from "node:fs";
-import {join} from "node:path";
+import {join} from "node:path/posix";
 import {Loader} from "./dataloader.js";
 import {isEnoent} from "./error.js";
 import {maybeStat} from "./files.js";

src/files.ts

@@ -1,17 +1,27 @@
-import {type Stats, existsSync} from "node:fs";
+import type {Stats} from "node:fs";
+import {existsSync} from "node:fs";
 import {mkdir, readdir, stat} from "node:fs/promises";
-import {dirname, extname, join, normalize, relative} from "node:path";
+import op from "node:path";
+import {extname, join, normalize, relative, sep} from "node:path/posix";
 import {cwd} from "node:process";
 import {fileURLToPath} from "node:url";
 import {isEnoent} from "./error.js";
 
+export function toOsPath(path: string): string {
+  return path.split(sep).join(op.sep);
+}
+
+export function fromOsPath(path: string): string {
+  return path.split(op.sep).join(sep);
+}
+
 /**
  * Returns the relative path from the current working directory to the given
  * Framework source file, such as "./src/client/search.js". This is typically
  * used to rollup JavaScript and style bundles for built-in modules.
  */
 export function getClientPath(entry: string): string {
-  const path = relative(cwd(), join(dirname(fileURLToPath(import.meta.url)), "..", entry));
+  const path = fromOsPath(op.relative(cwd(), op.join(fileURLToPath(import.meta.url), "..", "..", entry)));
   if (path.endsWith(".js") && !existsSync(path)) {
     const tspath = path.slice(0, -".js".length) + ".ts";
     if (existsSync(tspath)) return tspath;
@@ -56,7 +66,7 @@ export async function maybeStat(path: string): Promise<Stats | undefined> {
 
 /** Like recursive mkdir, but for the parent of the specified output. */
 export async function prepareOutput(outputPath: string): Promise<void> {
-  const outputDir = dirname(outputPath);
+  const outputDir = op.dirname(outputPath);
   if (outputDir === ".") return;
   await mkdir(outputDir, {recursive: true});
 }

src/javascript/files.ts

@@ -1,4 +1,4 @@
-import {extname} from "node:path";
+import {extname} from "node:path/posix";
 import type {CallExpression, MemberExpression, Node} from "acorn";
 import {ancestor, simple} from "acorn-walk";
 import {relativePath, resolveLocalPath} from "../path.js";

src/javascript/module.ts

@@ -1,6 +1,6 @@
 import {createHash} from "node:crypto";
 import {existsSync, readFileSync, statSync} from "node:fs";
-import {join, relative} from "node:path";
+import {join, relative} from "node:path/posix";
 import type {Program} from "acorn";
 import {Parser} from "acorn";
 import {Loader} from "../dataloader.js";

src/npm.ts

@@ -1,6 +1,6 @@
 import {existsSync} from "node:fs";
 import {mkdir, readFile, readdir, writeFile} from "node:fs/promises";
-import {dirname, join} from "node:path";
+import {dirname, join} from "node:path/posix";
 import type {CallExpression} from "acorn";
 import {Parser} from "acorn";
 import {simple} from "acorn-walk";

src/observableApiConfig.ts

@@ -1,6 +1,6 @@
 import fs from "node:fs/promises";
 import os from "node:os";
-import path from "node:path";
+import op from "node:path";
 import {CliError, isEnoent} from "./error.js";
 
 export interface ConfigEffects {
@@ -69,7 +69,7 @@ export async function getDeployConfig(
   sourceRoot: string,
   effects: ConfigEffects = defaultEffects
 ): Promise<DeployConfig> {
-  const deployConfigPath = path.join(effects.cwd(), sourceRoot, ".observablehq", "deploy.json");
+  const deployConfigPath = op.join(effects.cwd(), sourceRoot, ".observablehq", "deploy.json");
   let config: object | null = null;
   try {
     const content = await effects.readFile(deployConfigPath, "utf8");
@@ -93,8 +93,8 @@ export async function setDeployConfig(
   newConfig: DeployConfig,
   effects: ConfigEffects = defaultEffects
 ): Promise<void> {
-  const dir = path.join(effects.cwd(), sourceRoot, ".observablehq");
-  const deployConfigPath = path.join(dir, "deploy.json");
+  const dir = op.join(effects.cwd(), sourceRoot, ".observablehq");
+  const deployConfigPath = op.join(dir, "deploy.json");
   const oldConfig = (await getDeployConfig(sourceRoot)) || {};
   const merged = {...oldConfig, ...newConfig};
   await effects.mkdir(dir, {recursive: true});
@@ -104,13 +104,13 @@ export async function setDeployConfig(
 export async function loadUserConfig(
   effects: ConfigEffects = defaultEffects
 ): Promise<{configPath: string; config: UserConfig}> {
-  const homeConfigPath = path.join(effects.homedir(), userConfigName);
+  const homeConfigPath = op.join(effects.homedir(), userConfigName);
 
   function* pathsToTry(): Generator<string> {
-    let cursor = path.resolve(effects.cwd());
+    let cursor = op.resolve(effects.cwd());
     while (true) {
-      yield path.join(cursor, userConfigName);
-      const nextCursor = path.dirname(cursor);
+      yield op.join(cursor, userConfigName);
+      const nextCursor = op.dirname(cursor);
       if (nextCursor === cursor) break;
       cursor = nextCursor;
     }

src/path.ts

@@ -1,4 +1,4 @@
-import {dirname, join} from "node:path";
+import {dirname, join} from "node:path/posix";
 
 /**
  * Returns the normalized relative path from "/file/path/to/a" to

src/preview.ts

@@ -4,7 +4,7 @@ import type {FSWatcher, WatchEventType} from "node:fs";
 import {access, constants, readFile, stat} from "node:fs/promises";
 import {createServer} from "node:http";
 import type {IncomingMessage, RequestListener, Server, ServerResponse} from "node:http";
-import {basename, dirname, extname, join, normalize} from "node:path";
+import {basename, dirname, extname, join, normalize} from "node:path/posix";
 import {difference} from "d3-array";
 import type {PatchItem} from "fast-array-diff";
 import {getPatch} from "fast-array-diff";

src/rollup.ts

@@ -124,7 +124,7 @@ function importResolve(input: string, root: string, path: string): Plugin {
       ? {id: relativePath(path, "/_observablehq/stdlib/zip.js"), external: true} // TODO publish to npm
       : specifier.startsWith("npm:")
       ? {id: relativePath(path, await resolveNpmImport(root, specifier.slice("npm:".length))), external: true}
-      : !isPathImport(specifier) && !BUNDLED_MODULES.includes(specifier) // e.g., inputs.js imports "htl"
+      : !/^[a-z]:\\/i.test(specifier) && !isPathImport(specifier) && !BUNDLED_MODULES.includes(specifier) // e.g., inputs.js imports "htl"
       ? {id: relativePath(path, await resolveNpmImport(root, specifier)), external: true}
       : null;
   }

src/search.ts

@@ -1,4 +1,4 @@
-import {basename, join} from "node:path";
+import {basename, join} from "node:path/posix";
 import he from "he";
 import MiniSearch from "minisearch";
 import type {Config} from "./config.js";

src/telemetry.ts

@@ -1,7 +1,7 @@
 import {exec} from "node:child_process";
 import {createHash, randomUUID} from "node:crypto";
 import {readFile, writeFile} from "node:fs/promises";
-import {join} from "node:path";
+import {join} from "node:path/posix";
 import os from "os";
 import {CliError} from "./error.js";
 import type {Logger} from "./logger.js";

templates/default/package.json

@@ -2,8 +2,8 @@
   "type": "module",
   "private": true,
   "scripts": {
-    "clean": "rm -rf docs/.observablehq/cache",
-    "build": "rm -rf dist && observable build",
+    "clean": "rimraf docs/.observablehq/cache",
+    "build": "rimraf dist && observable build",
     "dev": "observable preview",
     "deploy": "observable deploy",
     "observable": "observable"
@@ -13,6 +13,9 @@
     "d3-dsv": "^3.0.1",
     "d3-time-format": "^4.1.0"
   },
+  "devDependencies": {
+    "rimraf": "^5.0.5"
+  },
   "engines": {
     "node": ">=20.6"
   }

templates/empty/package.json

@@ -2,8 +2,8 @@
   "type": "module",
   "private": true,
   "scripts": {
-    "clean": "rm -rf docs/.observablehq/cache",
-    "build": "rm -rf dist && observable build",
+    "clean": "rimraf docs/.observablehq/cache",
+    "build": "rimraf dist && observable build",
     "dev": "observable preview",
     "deploy": "observable deploy",
     "observable": "observable"
@@ -13,6 +13,9 @@
     "d3-dsv": "^3.0.1",
     "d3-time-format": "^4.1.0"
   },
+  "devDependencies": {
+    "rimraf": "^5.0.5"
+  },
   "engines": {
     "node": ">=20.6"
   }