page fragment loaders

docs/index.md

@@ -2,6 +2,18 @@
 index: false
 ---
 
+```js server echo
+console.log(`The current version of Node is: <b>${process.env.npm_package_version}.`);
+```
+
+```sh server echo
+uname -a
+```
+
+```sh server echo
+date
+```
+
 <style>
 
 .hero {

src/loader.ts

@@ -62,7 +62,7 @@ export interface LoaderOptions {
 
 export class LoaderResolver {
   private readonly root: string;
-  private readonly interpreters: Map<string, string[]>;
+  public readonly interpreters: Map<string, string[]>; // TODO cleaner
 
   constructor({root, interpreters}: {root: string; interpreters?: Record<string, string[] | null>}) {
     this.root = root;
@@ -81,7 +81,7 @@ export class LoaderResolver {
     const loader = this.findPage(path);
     if (!loader) throw enoent(path);
     const input = await readFile(join(this.root, await loader.load(options, effects)), "utf8");
-    return parseMarkdown(input, {source: loader.path, params: loader.params, ...options});
+    return await parseMarkdown(input, {source: loader.path, params: loader.params, ...options});
   }
 
   /**
@@ -213,7 +213,7 @@ export class LoaderResolver {
       const eext = fext.slice(0, -iext.length); // .zip
       const loader = new CommandLoader({
         command: command ?? commandPath,
-        args: params ? args.concat(defineParams(params)) : args,
+        args: withParams(args, params),
         path,
         params,
         root: this.root,
@@ -332,6 +332,10 @@ export class LoaderResolver {
   }
 }
 
+export function withParams(args: string[], params?: Params): string[] {
+  return params ? args.concat(defineParams(params)) : args;
+}
+
 function defineParams(params: Params): string[] {
   return Object.entries(params)
     .filter(([name]) => /^[a-z0-9_]+$/i.test(name)) // ignore non-ASCII parameters

src/markdown.ts

@@ -1,6 +1,7 @@
 /* eslint-disable import/no-named-as-default-member */
 import {createHash} from "node:crypto";
 import slugify from "@sindresorhus/slugify";
+import {spawn} from "cross-spawn";
 import he from "he";
 import MarkdownIt from "markdown-it";
 import type {RuleCore} from "markdown-it/lib/parser_core.mjs";
@@ -12,11 +13,12 @@ import type {Config} from "./config.js";
 import {mergeStyle} from "./config.js";
 import type {FrontMatter} from "./frontMatter.js";
 import {readFrontMatter} from "./frontMatter.js";
-import {html, rewriteHtmlPaths} from "./html.js";
+import {html, parseHtml, rewriteHtmlPaths} from "./html.js";
 import {parseInfo} from "./info.js";
 import {transformJavaScriptSync} from "./javascript/module.js";
 import type {JavaScriptNode} from "./javascript/parse.js";
 import {parseJavaScript} from "./javascript/parse.js";
+import {withParams} from "./loader.js";
 import {isAssetPath, relativePath} from "./path.js";
 import {parsePlaceholder} from "./placeholder.js";
 import type {Params} from "./route.js";
@@ -31,6 +33,12 @@ export interface MarkdownCode {
   mode: "inline" | "block" | "jsx";
 }
 
+export interface FragmentLoader {
+  id: string;
+  tag: string;
+  source: string;
+}
+
 export interface MarkdownPage {
   title: string | null;
   head: string | null;
@@ -46,6 +54,7 @@ export interface MarkdownPage {
 
 interface ParseContext {
   code: MarkdownCode[];
+  fragments: FragmentLoader[];
   startLine: number;
   currentLine: number;
   path: string;
@@ -119,14 +128,25 @@ function makeFenceRenderer(baseRenderer: RenderRule): RenderRule {
     let html = "";
     let source: string | undefined;
     try {
-      source = isFalse(attributes.run) ? undefined : getLiveSource(token.content, tag, attributes);
+      source =
+        attributes.server != null
+          ? token.content
+          : isFalse(attributes.run)
+          ? undefined
+          : getLiveSource(token.content, tag, attributes);
       if (source != null) {
+        let loading = false;
         const id = uniqueCodeId(context, source);
         // TODO const sourceLine = context.startLine + context.currentLine;
-        const node = parseJavaScript(source, {path, params});
-        context.code.push({id, node, mode: tag === "jsx" || tag === "tsx" ? "jsx" : "block"});
+        if (attributes.server != null) {
+          context.fragments.push({id, tag, source});
+        } else {
+          const node = parseJavaScript(source, {path, params});
+          context.code.push({id, node, mode: tag === "jsx" || tag === "tsx" ? "jsx" : "block"});
+          loading = node.expression;
+        }
         html += `<div class="observablehq observablehq--block">${
-          node.expression ? "<observablehq-loading></observablehq-loading>" : ""
+          loading ? "<observablehq-loading></observablehq-loading>" : ""
         }<!--:${id}:--></div>\n`;
       }
     } catch (error) {
@@ -214,6 +234,7 @@ export interface ParseOptions {
   path: string;
   style?: Config["style"];
   scripts?: Config["scripts"];
+  loaders?: Config["loaders"];
   head?: Config["head"];
   header?: Config["header"];
   footer?: Config["footer"];
@@ -243,14 +264,57 @@ export function createMarkdownIt({
   return markdownIt === undefined ? md : markdownIt(md);
 }
 
-export function parseMarkdown(input: string, options: ParseOptions): MarkdownPage {
+export async function parseMarkdown(input: string, options: ParseOptions): Promise<MarkdownPage> {
   const {md, path, source = path, params} = options;
   const {content, data} = readFrontMatter(input);
   const code: MarkdownCode[] = [];
-  const context: ParseContext = {code, startLine: 0, currentLine: 0, path, params};
+  const fragments: FragmentLoader[] = [];
+  const context: ParseContext = {code, fragments, startLine: 0, currentLine: 0, path, params};
   const tokens = md.parse(content, context);
-  const body = md.renderer.render(tokens, md.options, context); // Note: mutates code!
   const title = data.title !== undefined ? data.title : findTitle(tokens);
+  let body = md.renderer.render(tokens, md.options, context); // Note: mutates code!
+
+  // Rewrite body to render fragments.
+  if (fragments.length) {
+    const {document} = parseHtml(body);
+    const roots = findRoots(document, document.body);
+    const interpreters = options.loaders!.interpreters;
+    for (const fragment of fragments) {
+      const root: Comment = roots.get(fragment.id);
+      const [command, ...args] = interpreters.get(`.${fragment.tag}`)!;
+      let target = "";
+      const subprocess = spawn(
+        command,
+        withParams(
+          command === "sh"
+            ? args.concat("-s", "--") // TODO make this configurable
+            : args.concat("-"), // TODO make this configurable
+          params
+        ),
+        {
+          windowsHide: true,
+          stdio: ["pipe", "pipe", "inherit"]
+        }
+      );
+      subprocess.stdin.write(fragment.source);
+      subprocess.stdin.end();
+      subprocess.stdout.on("data", (data) => {
+        target += data.toString();
+      });
+      const code = await new Promise((resolve, reject) => {
+        subprocess.on("error", reject);
+        subprocess.on("close", resolve);
+      });
+      if (code !== 0) {
+        throw new Error(`loader exited with code ${code}`);
+      }
+      const template = document.createElement("template");
+      template.innerHTML = target;
+      root.replaceWith(template.content.cloneNode(true));
+    }
+    body = document.body.innerHTML;
+  }
+
   return {
     head: getHead(title, data, options),
     header: getHeader(title, data, options),
@@ -265,6 +329,22 @@ export function parseMarkdown(input: string, options: ParseOptions): MarkdownPag
   };
 }
 
+function findRoots(document, root) {
+  const roots = new Map();
+  const iterator = document.createNodeIterator(root, 128, null);
+  let node;
+  while ((node = iterator.nextNode())) {
+    if (isRoot(node)) {
+      roots.set(node.data.slice(1, -1), node);
+    }
+  }
+  return roots;
+}
+
+function isRoot(node) {
+  return node.nodeType === 8 && /^:[0-9a-f]{8}(?:-\d+)?:$/.test(node.data);
+}
+
 /** Like parseMarkdown, but optimized to return only metadata. */
 export function parseMarkdownMetadata(input: string, options: ParseOptions): Pick<MarkdownPage, "data" | "title"> {
   const {md, path} = options;

test/markdown-test.ts

@@ -23,7 +23,7 @@ describe("parseMarkdown(input)", () => {
 
     (only ? it.only : skip ? it.skip : it)(`test/input/${name}`, async () => {
       const source = await readFile(path, "utf8");
-      const snapshot = parseMarkdown(source, {path: name, md});
+      const snapshot = await parseMarkdown(source, {path: name, md});
       let allequal = true;
       for (const ext of ["html", "json"]) {
         const actual = ext === "json" ? jsonMeta(snapshot) : snapshot.body;

test/resolvers-test.ts

@@ -14,84 +14,84 @@ describe("getResolvers(page, {root, path})", () => {
   const builtins = ["observablehq:runtime", "observablehq:stdlib", "observablehq:client"];
   it("resolves directly-attached files", async () => {
     const options = getOptions({root: "test/input", path: "attached.md"});
-    const page = parseMarkdown("${FileAttachment('foo.csv')}", options);
+    const page = await parseMarkdown("${FileAttachment('foo.csv')}", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.files, new Set(["./foo.csv"]));
   });
   it("ignores files that are outside of the source root", async () => {
     const options = getOptions({root: "test/input", path: "attached.md"});
-    const page = parseMarkdown("${FileAttachment('../foo.csv')}", options);
+    const page = await parseMarkdown("${FileAttachment('../foo.csv')}", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.files, new Set([]));
   });
   it("detects file methods", async () => {
     const options = getOptions({root: "test/input", path: "attached.md"});
-    const page = parseMarkdown("${FileAttachment('foo.csv').csv}", options);
+    const page = await parseMarkdown("${FileAttachment('foo.csv').csv}", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["npm:d3-dsv", ...builtins]));
   });
   it("detects local static imports", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport './bar.js';\n```", options);
+    const page = await parseMarkdown("```js\nimport './bar.js';\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["./bar.js", ...builtins]));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./bar.js"]));
   });
   it("detects local transitive static imports", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport './other/foo.js';\n```", options);
+    const page = await parseMarkdown("```js\nimport './other/foo.js';\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["./other/foo.js", "./bar.js", ...builtins]));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./other/foo.js", "./bar.js"]));
   });
   it("detects local transitive static imports (2)", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport './transitive-static-import.js';\n```", options);
+    const page = await parseMarkdown("```js\nimport './transitive-static-import.js';\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["./transitive-static-import.js", "./other/foo.js", "./bar.js", ...builtins])); // prettier-ignore
     assert.deepStrictEqual(resolvers.localImports, new Set(["./transitive-static-import.js", "./other/foo.js", "./bar.js"])); // prettier-ignore
   });
   it("detects local transitive dynamic imports", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport './dynamic-import.js';\n```", options);
+    const page = await parseMarkdown("```js\nimport './dynamic-import.js';\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(["./dynamic-import.js", ...builtins]));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./dynamic-import.js", "./bar.js"]));
   });
   it("detects local transitive dynamic imports (2)", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport('./dynamic-import.js');\n```", options);
+    const page = await parseMarkdown("```js\nimport('./dynamic-import.js');\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./dynamic-import.js", "./bar.js"]));
   });
   it("detects local transitive dynamic imports (3)", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport('./transitive-dynamic-import.js');\n```", options);
+    const page = await parseMarkdown("```js\nimport('./transitive-dynamic-import.js');\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./transitive-dynamic-import.js", "./other/foo.js", "./bar.js"])); // prettier-ignore
   });
   it("detects local transitive dynamic imports (4)", async () => {
     const options = getOptions({root: "test/input/imports", path: "attached.md"});
-    const page = parseMarkdown("```js\nimport('./transitive-static-import.js');\n```", options);
+    const page = await parseMarkdown("```js\nimport('./transitive-static-import.js');\n```", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./transitive-static-import.js", "./other/foo.js", "./bar.js"])); // prettier-ignore
   });
   it("detects local dynamic imports", async () => {
     const options = getOptions({root: "test/input", path: "attached.md"});
-    const page = parseMarkdown("${import('./foo.js')}", options);
+    const page = await parseMarkdown("${import('./foo.js')}", options);
     const resolvers = await getResolvers(page, options);
     assert.deepStrictEqual(resolvers.staticImports, new Set(builtins));
     assert.deepStrictEqual(resolvers.localImports, new Set(["./foo.js"]));
   });
 });
 
 describe("resolveLink(href) with {preserveExtension: true}", () => {
-  const options = getOptions({root: "test/input", path: "sub/index.html", preserveExtension: true});
-  const page = parseMarkdown("", options);
   async function getResolveLink() {
+    const options = getOptions({root: "test/input", path: "sub/index.html", preserveExtension: true});
+    const page = await parseMarkdown("", options);
     const resolvers = await getResolvers(page, options);
     return resolvers.resolveLink;
   }
@@ -164,9 +164,9 @@ describe("resolveLink(href) with {preserveExtension: true}", () => {
 });
 
 describe("resolveLink(href) with {preserveExtension: false}", () => {
-  const options = getOptions({root: "test/input", path: "sub/index.html", preserveExtension: false});
-  const page = parseMarkdown("", options);
   async function getResolveLink() {
+    const options = getOptions({root: "test/input", path: "sub/index.html", preserveExtension: false});
+    const page = await parseMarkdown("", options);
     const resolvers = await getResolvers(page, options);
     return resolvers.resolveLink;
   }