diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f37b8a6..fc04296 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -102,7 +102,7 @@ Long-term, Boutiques shifts from being the primary frontend to primarily a **bac - **Boutiques** - remains as both a frontend and a backend - **Serialized Python argparse** (`argdump`) - implemented - **Connectome Workbench** (`workbench`) - implemented; covers NiWrap's `wb_command` suite -- **Custom TypeScript-types-like language** (planned) - the intended primary way to define CLI specs +- **argtype** (`argtype`) - implemented as both a frontend and a backend; the hand-authored, TypeScript-types-like DSL (see the [argtype spec](https://nx10.dev/argtype/)) intended as the primary way to define CLI specs. Covers the core grammar plus the `outputs`, `mediatypes`, and `paths` (`.mutable()` / `.resolveParent()`) extensions; `set` lowers to a sequence, `any` to its first branch, and the draft `constraints` extension is parsed-and-ignored. Frontend lives in `frontend/argtype/`, backend (IR -> argtype source, the round-trip/dogfooding path) in `backend/argtype/` ## Ecosystem Context diff --git a/packages/cli/src/backends.test.ts b/packages/cli/src/backends.test.ts index 2327899..14544a2 100644 --- a/packages/cli/src/backends.test.ts +++ b/packages/cli/src/backends.test.ts @@ -7,9 +7,16 @@ describe("backends", () => { expect(knownBackends).toContain("python"); expect(knownBackends).toContain("typescript"); expect(knownBackends).toContain("boutiques"); + expect(knownBackends).toContain("argtype"); expect(knownBackends).toContain("schema"); }); + it("resolves the argtype serialization backend", () => { + const { backends, unknown } = resolveBackends(["argtype"]); + expect(unknown).toEqual([]); + expect(backends.map((b) => b.name)).toEqual(["argtype"]); + }); + it("resolves aliases to backend instances", () => { const { backends, unknown } = resolveBackends(["python", "typescript"]); expect(unknown).toEqual([]); diff --git a/packages/cli/src/backends.ts b/packages/cli/src/backends.ts index 3b95cf1..9fd2ebb 100644 --- a/packages/cli/src/backends.ts +++ b/packages/cli/src/backends.ts @@ -1,4 +1,5 @@ import { + ArgtypeBackend, BoutiquesBackend, JsonSchemaBackend, NipypeBackend, @@ -19,6 +20,7 @@ const registry: Record Backend> = { schema: () => new JsonSchemaBackend(), "json-schema": () => new JsonSchemaBackend(), boutiques: () => new BoutiquesBackend(), + argtype: () => new ArgtypeBackend(), nipype: () => new NipypeBackend(), pydra: () => new PydraBackend(), }; diff --git a/packages/cli/src/build.ts b/packages/cli/src/build.ts index 7dc1c0f..45781ae 100644 --- a/packages/cli/src/build.ts +++ b/packages/cli/src/build.ts @@ -26,6 +26,7 @@ const SUPPORTED_FORMATS: ReadonlySet = new Set([ "argdump", "workbench", "mrtrix", + "argtype", ]); export interface BuildOptions { diff --git a/packages/core/src/backend/argtype/__fixtures__/microsyntax.argtype b/packages/core/src/backend/argtype/__fixtures__/microsyntax.argtype new file mode 100644 index 0000000..29e6a11 --- /dev/null +++ b/packages/core/src/backend/argtype/__fixtures__/microsyntax.argtype @@ -0,0 +1,15 @@ +--- +exe: "cut" +version: "9.0" +extensions: + - mediatypes +--- + +/// Cut fields from each line, exercising join micro-syntax and media types +cut: seq( + input: path.mediaType("text/plain"), + seq("--fields=", fields: rep(int).join(",")).join(), + opt("-d", delimiter: str), + defines: rep(seq(str, "=", str).join("")), + out: path.mediaType("application/json").mediaType("text/csv"), +) diff --git a/packages/core/src/backend/argtype/__fixtures__/subcommands.argtype b/packages/core/src/backend/argtype/__fixtures__/subcommands.argtype new file mode 100644 index 0000000..b2e1789 --- /dev/null +++ b/packages/core/src/backend/argtype/__fixtures__/subcommands.argtype @@ -0,0 +1,25 @@ +--- +exe: "vcs" +version: "1.2.0" +authors: + - "VCS Authors" +--- + +/// Mini version control with subcommands +vcs: alt( + /// Record changes to the repository + commit: seq( + "commit", + opt("-m", message: str), + opt("--amend"), + opt("--author", author: str), + ), + /// Publish local commits to a remote + push: seq( + "push", + opt("-u", upstream: str), + opt("--force"), + remote: str = "origin", + branch: str, + ), +) diff --git a/packages/core/src/backend/argtype/argtype.ts b/packages/core/src/backend/argtype/argtype.ts new file mode 100644 index 0000000..df629c1 --- /dev/null +++ b/packages/core/src/backend/argtype/argtype.ts @@ -0,0 +1,45 @@ +import type { CodegenContext } from "../../manifest/index.js"; +import type { Backend, EmittedApp } from "../backend.js"; +import { Scope } from "../scope.js"; +import { snakeCase } from "../string-case.js"; +import { generateArgtype } from "./emit.js"; + +/** + * A per-tool file stem, so co-located tools in a catalog build don't clobber one + * another's `descriptor.argtype`. Standalone single-tool builds (no scope) keep + * the bare name. Mirrors the JSON Schema backend's `schemaStem`. + */ +function argtypeStem(ctx: CodegenContext, scope?: Scope): string | undefined { + const id = ctx.app?.id; + if (!id || !scope) return undefined; + return scope.add(snakeCase(id) || "descriptor"); +} + +/** + * Serialization backend: emit argtype sugar-DSL source from the IR + `AppMeta`. + * + * The dogfooding / round-trip counterpart to the argtype frontend. Like the + * Boutiques backend it only needs the IR (`ctx.expr`) and app metadata + * (`ctx.app`), never the solved bindings, so it ignores the rest of the context. + */ +export class ArgtypeBackend implements Backend { + readonly name = "argtype"; + readonly target = "argtype"; + + /** One scope per package so per-tool file stems stay unique in the suite dir. */ + newPackageScope(): Scope { + return new Scope(); + } + + emitApp(ctx: CodegenContext, scope?: Scope): EmittedApp { + const { source, warnings } = generateArgtype(ctx.expr, ctx.app); + const stem = argtypeStem(ctx, scope); + const filename = stem ? `${stem}.argtype` : "descriptor.argtype"; + return { + meta: ctx.app, + files: new Map([[filename, source]]), + errors: [], + warnings, + }; + } +} diff --git a/packages/core/src/backend/argtype/corpus-roundtrip.test.ts b/packages/core/src/backend/argtype/corpus-roundtrip.test.ts new file mode 100644 index 0000000..fc5ff85 --- /dev/null +++ b/packages/core/src/backend/argtype/corpus-roundtrip.test.ts @@ -0,0 +1,53 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { compile } from "../../index.js"; +import { ArgtypeParser } from "../../frontend/argtype/parser-frontend.js"; +import { generateArgtype } from "./emit.js"; + +/** + * Corpus validity guard: for every descriptor in the typecheck catalog (the + * curated hard cases - unions, dup outputs/variants, mutable inputs, mrtrix, + * workbench), the emitter must produce argtype source that re-parses with ZERO + * errors. This is the strongest property that holds universally: it proves the + * backend never emits invalid/non-reparseable syntax across the shape diversity + * of real tools, independent of the documented annotation lossiness (docs, + * titles, media types, mutable) and the frontend's outputs-collected-to-root + * behavior, which prevent exact codegen equality for some tools. + */ + +const CATALOG = new URL( + "../../../../../packages/cli/test-fixtures/typecheck-catalog/suite/1.0/", + import.meta.url, +); + +const TOOLS: Array<[string, string]> = [ + ["DenoiseImage", "boutiques.json"], + ["antsApplyTransforms", "boutiques.json"], + ["borderMerge", "workbench.json"], + ["dupOutputs", "boutiques.json"], + ["dupVariant", "boutiques.json"], + ["dwi2response", "boutiques.json"], + ["mrtrixDemo", "mrtrix.json"], + ["mutate", "boutiques.json"], + ["shapes", "boutiques.json"], + ["variousTypes", "boutiques.json"], +]; + +const argtype = new ArgtypeParser(); + +describe("argtype backend: corpus emits re-parseable source", () => { + for (const [tool, file] of TOOLS) { + it(`${tool}: emitted argtype re-parses cleanly`, () => { + const source = readFileSync(fileURLToPath(new URL(`${tool}/${file}`, CATALOG)), "utf8"); + const direct = compile(source); + expect(direct.errors, `${tool}: direct parse errors`).toEqual([]); + + const { source: emitted } = generateArgtype(direct.expr, direct.meta); + const reparsed = argtype.parse(emitted); + expect(reparsed.errors, `${tool}: emitted argtype failed to re-parse:\n${emitted}`).toEqual( + [], + ); + }); + } +}); diff --git a/packages/core/src/backend/argtype/emit.test.ts b/packages/core/src/backend/argtype/emit.test.ts new file mode 100644 index 0000000..68add73 --- /dev/null +++ b/packages/core/src/backend/argtype/emit.test.ts @@ -0,0 +1,322 @@ +import { describe, expect, it } from "vitest"; +import type { AppMeta } from "../../ir/meta.js"; +import type { Expr, Path, Repeat } from "../../ir/node.js"; +import { int, lit, opt, seq } from "../../ir/builders.js"; +import { createContext } from "../../manifest/context.js"; +import { ArgtypeParser } from "../../frontend/argtype/parser-frontend.js"; +import { ArgtypeBackend } from "./argtype.js"; +import { generateArgtype } from "./emit.js"; + +describe("argtype emitter: frontmatter", () => { + it("emits container, stream outputs, and both extensions", () => { + const app: AppMeta = { + id: "tool", + version: "1.0", + doc: { authors: ["A One", "A Two"], urls: ["https://example.org"] }, + container: { image: "docker://tool:1.0", type: "docker" }, + stdout: { name: "log", doc: { description: "stdout log" } }, + }; + const path: Path = { kind: "path", attrs: { mediaTypes: ["image/png"] }, meta: { name: "in" } }; + // The leading `tool` literal is the executable, stripped into `exe:`. + const expr = seq(lit("tool"), path); + expr.meta = { + name: "tool", + outputs: [{ name: "o", tokens: [{ kind: "literal", value: "out.png" }] }], + }; + + const { source, warnings } = generateArgtype(expr, app); + expect(warnings).toEqual([]); + expect(source).toContain(`exe: "tool"`); + expect(source).toContain("container:"); + expect(source).toContain(` image: "docker://tool:1.0"`); + expect(source).toContain(` type: "docker"`); + expect(source).toContain("stdout:"); + expect(source).toContain(` name: "log"`); + expect(source).toContain(` description: "stdout log"`); + expect(source).toContain("extensions:"); + expect(source).toContain(" - outputs"); + expect(source).toContain(" - mediatypes"); + expect(source).toContain(`.mediaType("image/png")`); + }); + + it("emits the app title as a root doc block and warns for truly unsupported doc fields", () => { + const app: AppMeta = { + id: "tool", + doc: { + title: "Title", + description: "Long description.", + literature: ["doi:1"], + comment: "note", + urls: ["a", "b"], + }, + }; + const { source, warnings } = generateArgtype(seq(lit("tool")), app); + // Title + description round-trip as a `///` block (# title, blank, description). + expect(source).toContain("/// # Title\n///\n/// Long description."); + // authors, urls (list), references round-trip through frontmatter. + expect(source).toContain("urls:"); + expect(source).toContain(` - "a"`); + expect(source).toContain(` - "b"`); + expect(source).toContain("references:"); + expect(source).toContain(` - "doi:1"`); + const messages = warnings.map((w) => w.message).join("\n"); + // title / literature / urls are all now representable; only `comment` isn't. + expect(messages).not.toContain("title"); + expect(messages).not.toContain("literature"); + expect(messages).not.toContain("url"); + expect(messages).toContain("comment"); + }); +}); + +describe("argtype emitter: IR features with no surface", () => { + it("emits mutable / resolveParent as the paths extension and round-trips", () => { + const path: Path = { + kind: "path", + attrs: { mutable: true, resolveParent: true }, + meta: { name: "workdir" }, + }; + const expr = seq(lit("tool"), path); + expr.meta = { name: "tool" }; + const { source, warnings } = generateArgtype(expr, { id: "tool" }); + expect(warnings).toEqual([]); + expect(source).toContain("workdir: path.mutable().resolveParent()"); + expect(source).toContain(" - paths"); // extension declared in frontmatter + // Round-trips: re-parsing recovers the same path attrs. + const reparsed = new ArgtypeParser().parse(source); + expect(reparsed.errors).toEqual([]); + const seqNode = reparsed.expr; + const pathNode = + seqNode.kind === "sequence" ? seqNode.attrs.nodes.find((n) => n.kind === "path") : undefined; + expect(pathNode?.kind === "path" && pathNode.attrs.mutable).toBe(true); + expect(pathNode?.kind === "path" && pathNode.attrs.resolveParent).toBe(true); + }); + + it("emits a one-sided repeat count via countMin/countMax", () => { + const rep: Repeat = { + kind: "repeat", + attrs: { node: { kind: "int", attrs: {} }, countMax: 2 }, + meta: { name: "vals" }, + }; + const { source, warnings } = generateArgtype(seq(lit("tool"), rep), { id: "tool" }); + expect(source).toContain("rep(int).countMax(2)"); + expect(source).not.toContain(".count("); + expect(warnings).toEqual([]); + }); +}); + +describe("argtype emitter: micro-syntax and counts", () => { + it("emits .join() for an empty separator and .count(n) for a fixed count", () => { + const rep: Repeat = { + kind: "repeat", + attrs: { node: { kind: "float", attrs: {} }, countMin: 3, countMax: 3 }, + meta: { name: "coord" }, + }; + const joined = seq(lit("--x="), { kind: "int", attrs: {} }); + joined.attrs.join = ""; + const expr = seq(joined, rep); + expr.meta = { name: "tool" }; + const { source } = generateArgtype(expr, { id: "tool" }); + expect(source).toContain(".join()"); + expect(source).toContain(".count(3)"); + }); +}); + +describe("argtype emitter: name/ref consistency and escaping", () => { + it("emits a non-identifier output ref as a quoted name matching its quoted label", () => { + // A Boutiques-style id with a dash is not a valid identifier; the label and + // the template `{ref}` are both quoted (verbatim), so the ref still resolves + // and the name survives the round-trip exactly (no lossy sanitization). + const path: Path = { kind: "path", attrs: {}, meta: { name: "out-file" } }; + const expr = seq(lit("tool"), path); + expr.meta = { + name: "tool", + outputs: [ + { + name: "result", + tokens: [{ kind: "ref", target: { kind: "node-ref", name: "out-file" } }], + }, + ], + }; + const { source, warnings } = generateArgtype(expr, { id: "tool" }); + expect(warnings).toEqual([]); // quoted, not sanitized -> no rename warning + expect(source).toContain(`"out-file": path`); + expect(source).toContain('`{"out-file"}`'); + // The exact dashed name survives re-parse on both the node and the ref. + const r = new ArgtypeParser().parse(source); + expect(r.errors).toEqual([]); + const inner = + r.expr.kind === "sequence" ? r.expr.attrs.nodes.find((n) => n.kind === "path") : undefined; + expect(inner?.meta?.name).toBe("out-file"); + const output = r.expr.meta?.outputs?.[0]; + const ref = output?.tokens.find((t) => t.kind === "ref"); + expect(ref?.kind === "ref" ? ref.target.name : undefined).toBe("out-file"); + }); + + it("quotes non-identifier names as quoted labels and round-trips them exactly", () => { + const flag: Path = { kind: "path", attrs: {}, meta: { name: "1D" } }; + const expr = seq(lit("1deval"), flag); + expr.meta = { name: "1deval" }; + const { source, warnings } = generateArgtype(expr, { id: "1deval" }); + expect(warnings).toEqual([]); // quoted, not sanitized -> no rename warning + expect(source).toContain(`"1deval": seq(`); + expect(source).toContain(`"1D": path`); + // Names survive the round-trip exactly (no `_1D` sanitization). + const r = new ArgtypeParser().parse(source); + expect(r.errors).toEqual([]); + expect(r.expr.meta?.name).toBe("1deval"); + const inner = + r.expr.kind === "sequence" ? r.expr.attrs.nodes.find((n) => n.kind === "path") : undefined; + expect(inner?.meta?.name).toBe("1D"); + }); + + it("escapes literal braces/backticks in an output path and round-trips them", () => { + const expr = seq(lit("tool")); + expr.meta = { + name: "tool", + outputs: [{ name: "o", tokens: [{ kind: "literal", value: "{OUT_DIR}/a`b.txt" }] }], + }; + const { source, warnings } = generateArgtype(expr, { id: "tool" }); + expect(warnings).toEqual([]); + expect(source).toContain("\\{OUT_DIR\\}/a\\`b.txt"); + // Re-parsing recovers the exact literal path (braces/backtick are literal). + const r = new ArgtypeParser().parse(source); + expect(r.errors).toEqual([]); + const out = r.expr.meta?.outputs?.find((o) => o.name === "o"); + expect(out?.tokens).toEqual([{ kind: "literal", value: "{OUT_DIR}/a`b.txt" }]); + }); + + it("round-trips a frontmatter value containing a quote", () => { + const expr = seq(lit("tool")); + expr.meta = { name: "tool" }; + const { source, warnings } = generateArgtype(expr, { id: "tool", version: 'v"1' }); + expect(warnings).toEqual([]); + const r = new ArgtypeParser().parse(source); + expect(r.errors).toEqual([]); + expect(r.meta?.version).toBe('v"1'); + }); + + it("warns for a divergent union discriminator", () => { + const armA = seq(lit("a")); + armA.meta = { name: "alpha", variantTag: "AlphaVariant" }; + const armB = seq(lit("b")); + armB.meta = { name: "beta" }; + const alt: Expr = { + kind: "alternative", + attrs: { alts: [armA, armB] }, + meta: { name: "mode" }, + }; + const expr = seq(lit("tool"), alt); + expr.meta = { name: "tool" }; + const { warnings } = generateArgtype(expr, { id: "tool" }); + const messages = warnings.map((w) => w.message).join("\n"); + expect(messages).toContain("AlphaVariant"); // variantTag != name warned + }); +}); + +describe("ArgtypeBackend", () => { + function stubCtx(app: { id: string }) { + const expr: Expr = seq(lit(app.id), opt(lit("-v"), "verbose")); + expr.meta = { name: app.id }; + // The argtype backend only reads ctx.expr + ctx.app; the solver fields are + // unused, so a minimal stub context is enough to exercise emitApp. + return createContext( + expr, + { bindings: new Map(), resolve: () => undefined } as never, + { scopes: [], diagnostics: { errors: [], warnings: [] } } as never, + { app }, + ); + } + + it("emits a descriptor.argtype file from a CodegenContext", () => { + const result = new ArgtypeBackend().emitApp(stubCtx({ id: "tool" })); + expect(result.errors).toEqual([]); + const file = result.files.get("descriptor.argtype"); + expect(file).toBeDefined(); + expect(file).toContain("tool: seq("); + expect(file).toContain(`verbose: opt("-v")`); + }); + + it("uses per-tool file stems within a package scope so co-located tools don't collide", () => { + const backend = new ArgtypeBackend(); + const scope = backend.newPackageScope(); + const a = backend.emitApp(stubCtx({ id: "greet" }), scope); + const b = backend.emitApp(stubCtx({ id: "farewell" }), scope); + expect([...a.files.keys()]).toEqual(["greet.argtype"]); + expect([...b.files.keys()]).toEqual(["farewell.argtype"]); + }); +}); + +describe("argtype emitter: doc round-trip and non-finite numbers", () => { + const innerPath = (r: ReturnType) => + r.expr.kind === "sequence" ? r.expr.attrs.nodes.find((n) => n.kind === "path") : undefined; + + it("emits a title-less `# ...` description as chaining so it is not promoted to a title", () => { + const p: Path = { + kind: "path", + attrs: {}, + meta: { name: "in", doc: { description: "# leading hash\nmore text" } }, + }; + const expr = seq(lit("tool"), p); + expr.meta = { name: "tool" }; + const { source } = generateArgtype(expr); + // Chaining (verbatim), not a `///` block that would re-read `# ...` as a title. + expect(source).toContain(`.description(`); + const r = new ArgtypeParser().parse(source); + expect(r.errors).toEqual([]); + expect(innerPath(r)?.meta?.doc?.description).toBe("# leading hash\nmore text"); + expect(innerPath(r)?.meta?.doc?.title).toBeUndefined(); + }); + + it("emits a multi-line title as chaining so all lines survive", () => { + const p: Path = { + kind: "path", + attrs: {}, + meta: { name: "in", doc: { title: "Line one\nLine two", description: "body" } }, + }; + const expr = seq(lit("tool"), p); + expr.meta = { name: "tool" }; + const { source } = generateArgtype(expr); + expect(source).toContain(`.title(`); + const r = new ArgtypeParser().parse(source); + expect(r.errors).toEqual([]); + expect(innerPath(r)?.meta?.doc?.title).toBe("Line one\nLine two"); + expect(innerPath(r)?.meta?.doc?.description).toBe("body"); + }); + + it("round-trips an output-entry `# ...` description via chaining, not a `///` block", () => { + const p: Path = { kind: "path", attrs: {}, meta: { name: "x" } }; + const expr = seq(lit("tool"), p); + expr.meta = { + name: "tool", + outputs: [ + { + name: "mask", + doc: { description: "# produced\nthe mask file" }, + tokens: [ + { kind: "literal", value: "" }, + { kind: "ref", target: { kind: "node-ref", name: "x" } }, + { kind: "literal", value: ".nii" }, + ], + }, + ], + }; + const { source } = generateArgtype(expr); + const r = new ArgtypeParser().parse(source); + expect(r.errors).toEqual([]); + const out = r.expr.meta?.outputs?.[0]; + expect(out?.doc?.description).toBe("# produced\nthe mask file"); + expect(out?.doc?.title).toBeUndefined(); + }); + + it("drops a non-finite value bound with a warning instead of emitting an un-parseable token", () => { + const n = int(); + n.attrs.maxValue = Infinity; + n.meta = { name: "n" }; + const expr = seq(lit("tool"), n); + expr.meta = { name: "tool" }; + const { source, warnings } = generateArgtype(expr); + expect(source).not.toContain("Infinity"); + expect(warnings.some((w) => /Non-finite/.test(w.message))).toBe(true); + expect(new ArgtypeParser().parse(source).errors).toEqual([]); + }); +}); diff --git a/packages/core/src/backend/argtype/emit.ts b/packages/core/src/backend/argtype/emit.ts new file mode 100644 index 0000000..d4cc3c1 --- /dev/null +++ b/packages/core/src/backend/argtype/emit.ts @@ -0,0 +1,680 @@ +/** + * argtype backend: emit argtype sugar-DSL source text from Styx IR + `AppMeta`. + * + * This is the inverse of the argtype frontend's lowering (`frontend/argtype/ + * lower.ts`): it walks the `Expr` tree and prints surface syntax, mirroring each + * lowering decision in reverse. Unlike the typed-language backends it needs only + * the IR and `AppMeta`, never the solver, so it is a pure pretty-printer. + * + * Intentional lossiness (the frontend already collapses these, so the IR holds + * no record of them and emitting the collapsed form is correct): + * - `set` -> `sequence` and `any` -> first branch: the IR has no such nodes, so + * we emit `seq` / `alt`. + * - Output ops `strip_prefix` / `basename` and output-level `.or()` are not in + * the IR's `OutputToken`, so there is nothing to emit. + * + * `path.attrs.mutable` / `resolveParent` emit as the `paths` extension's + * `.mutable()` / `.resolveParent()`. A `doc.title` + `doc.description` emit as a + * `///` block using the summary-line convention (title, blank `///`, description). + * + * IR features with no argtype surface are handled explicitly (always a backend + * warning): output media types, stream `doc.title`, + * `AppMeta.doc.literature/comment`, and a union arm's `variantTag` when it + * differs from the arm name (the frontend re-derives the `@type` discriminator + * from the name on re-parse). + */ + +import type { AppMeta, Output, StreamOutput } from "../../ir/meta.js"; +import type { Documentation } from "../../ir/types.js"; +import type { Expr, Optional, Repeat } from "../../ir/node.js"; +import type { EmitWarning } from "../backend.js"; + +const INDENT = " "; + +function pad(level: number): string { + return INDENT.repeat(level); +} + +function num(n: number): string { + return String(n); +} + +/** Escape a string for a double-quoted argtype literal (matches the lexer). */ +function escapeString(s: string): string { + return s + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"') + .replace(/\n/g, "\\n") + .replace(/\t/g, "\\t") + .replace(/\r/g, "\\r"); +} + +function quote(s: string): string { + return `"${escapeString(s)}"`; +} + +/** A value terminal kind whose `meta.defaultValue` argtype can express. */ +function isValueTerminal( + node: Expr, +): node is Extract { + return ( + node.kind === "int" || node.kind === "float" || node.kind === "str" || node.kind === "path" + ); +} + +/** + * Find the single value terminal a wrapper's default should attach to. Descends + * through optional/repeat and a sequence with exactly one non-literal child + * (the flag-value pattern `seq(lit, value)`). Returns undefined when the value + * slot is ambiguous (multiple non-literal children) or absent (a bare literal + * flag), in which case the wrapper default has no terminal to sink onto. + */ +function findValueTerminal(node: Expr): Expr | undefined { + switch (node.kind) { + case "int": + case "float": + case "str": + case "path": + return node; + case "optional": + return findValueTerminal(node.attrs.node); + case "repeat": + return findValueTerminal(node.attrs.node); + case "sequence": { + const nonLiteral = node.attrs.nodes.filter((n) => n.kind !== "literal"); + return nonLiteral.length === 1 ? findValueTerminal(nonLiteral[0]!) : undefined; + } + default: + return undefined; + } +} + +/** + * Push a wrapper's `meta.defaultValue` down onto its inner value terminal so it + * can be emitted as `int = 5` / `.default(...)`. Boutiques (and argparse) hoist + * an input's default onto the outermost wrapper (`optional` / `sequence`); the + * argtype surface only carries `.default()` on a terminal, so we relocate it. + * Boolean defaults are left in place: those are the `opt(literal)` flag-false + * convention, which the frontend regenerates for free and which has no terminal. + */ +function sinkDefaults(expr: Expr): Expr { + const clone = structuredClone(expr); + const visit = (node: Expr): void => { + switch (node.kind) { + case "optional": + visit(node.attrs.node); + sinkInto(node); + break; + case "repeat": + visit(node.attrs.node); + sinkInto(node); + break; + case "sequence": + for (const c of node.attrs.nodes) visit(c); + sinkInto(node); + break; + case "alternative": + for (const c of node.attrs.alts) visit(c); + break; + default: + break; + } + }; + visit(clone); + return clone; +} + +function sinkInto(wrapper: Expr): void { + const dv = wrapper.meta?.defaultValue; + if (dv === undefined || typeof dv === "boolean") return; + const terminal = findValueTerminal(wrapper); + if (!terminal || !isValueTerminal(terminal) || terminal.meta?.defaultValue !== undefined) return; + terminal.meta = { ...terminal.meta, defaultValue: dv }; + const meta = { ...wrapper.meta }; + delete meta.defaultValue; + wrapper.meta = Object.keys(meta).length > 0 ? meta : undefined; +} + +/** + * Detect the synthetic single-child sequence the frontend wraps a non-sequence + * root in. Such a wrapper carries only `name` (matching the child's name) plus + * any collected `outputs`; the doc/default live on the inner node. Returns the + * inner node and the wrapper's outputs so the caller can emit the inner node as + * the definition body (the frontend re-wraps it identically on re-parse). + */ +function syntheticWrap(root: Expr): { child: Expr; outputs?: Output[] } | undefined { + if (root.kind !== "sequence" || root.attrs.nodes.length !== 1 || root.attrs.join !== undefined) { + return undefined; + } + const meta = root.meta; + if (meta?.doc || meta?.defaultValue !== undefined || meta?.variantTag !== undefined) { + return undefined; + } + const child = root.attrs.nodes[0]!; + if (child.kind === "sequence") return undefined; // the frontend only wraps non-sequences + if ((child.meta?.name ?? undefined) !== (meta?.name ?? undefined)) return undefined; + return { child, ...(meta?.outputs && { outputs: meta.outputs }) }; +} + +function hasOutputs(expr: Expr): boolean { + return walkSome(expr, (n) => (n.meta?.outputs?.length ?? 0) > 0); +} + +function hasMediaTypes(expr: Expr): boolean { + return walkSome(expr, (n) => n.kind === "path" && (n.attrs.mediaTypes?.length ?? 0) > 0); +} + +function hasPaths(expr: Expr): boolean { + return walkSome(expr, (n) => n.kind === "path" && (!!n.attrs.mutable || !!n.attrs.resolveParent)); +} + +function walkSome(node: Expr, pred: (n: Expr) => boolean): boolean { + if (pred(node)) return true; + switch (node.kind) { + case "sequence": + return node.attrs.nodes.some((n) => walkSome(n, pred)); + case "alternative": + return node.attrs.alts.some((n) => walkSome(n, pred)); + case "optional": + return walkSome(node.attrs.node, pred); + case "repeat": + return walkSome(node.attrs.node, pred); + default: + return false; + } +} + +const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]*$/; + +/** A name rendered where an identifier is expected: bare when it is a valid + * identifier, otherwise a double-quoted form preserving it exactly. Used for + * both `label:` prefixes and output-template `{ref}` targets (templates accept a + * quoted name, `{"4d_output"}`), so the two always agree and no lossy + * sanitization is needed. */ +function identOrQuoted(name: string): string { + return IDENT_RE.test(name) ? name : quote(name); +} + +class ArgtypeEmitter { + readonly warnings: EmitWarning[] = []; + + private warn(message: string): void { + this.warnings.push({ message }); + } + + /** Render a name as it appears before a `:` label (bare identifier or quoted + * label, e.g. `"1deval":`, `"1D":`). */ + private labelFor(name: string): string { + return identOrQuoted(name); + } + + /** + * `///` doc lines for a `Documentation`. A title is written as a leading + * Markdown H1 (`# Title`); the description follows after a blank `///` line. + * Either alone emits just that part. + */ + private docLines(doc: Documentation | undefined): string[] { + if (!doc) return []; + const out: string[] = []; + const push = (text: string): void => { + for (const line of text.split("\n")) out.push(`/// ${line}`); + }; + if (doc.title) push(`# ${doc.title}`); + if (doc.title && doc.description) out.push("///"); + if (doc.description) push(doc.description); + return out; + } + + /** + * Whether a `///` block would be misread by the title convention on re-parse, + * so the doc must instead be emitted as `.title()` / `.description()` chaining + * (which sets the fields verbatim). Two cases: a multi-line title (only its + * first line would survive `splitDocText`), and a title-less description whose + * first line looks like an H1 heading (`# ...`, which would be promoted to a + * spurious title). A description under a title is safe - `splitDocText` only + * consumes the very first line as the title. + */ + private docNeedsChain(doc: Documentation | undefined): boolean { + if (!doc) return false; + if (doc.title?.includes("\n")) return true; + if (!doc.title && doc.description) { + return (doc.description.split("\n")[0] ?? "").trim().startsWith("# "); + } + return false; + } + + /** `.title(...)` / `.description(...)` chaining for a doc that cannot round-trip + * as a `///` block (see `docNeedsChain`). */ + private docChain(doc: Documentation | undefined): string { + if (!doc) return ""; + let chain = ""; + if (doc.title) chain += `.title(${quote(doc.title)})`; + if (doc.description) chain += `.description(${quote(doc.description)})`; + return chain; + } + + /** + * Warn for `Documentation` fields a node's `///` comment cannot carry. `title` + * and `description` are emitted (see `docLines`); literature / comment / + * authors / urls attached to an inner node have no surface, so surface them + * rather than dropping silently. + */ + private warnUnrepresentableDoc( + doc: + | { + literature?: string[]; + comment?: string; + authors?: string[]; + urls?: string[]; + } + | undefined, + where: string, + ): void { + if (!doc) return; + const lost: string[] = []; + if (doc.literature?.length) lost.push("literature"); + if (doc.comment) lost.push("comment"); + if (doc.authors?.length) lost.push("authors"); + if (doc.urls?.length) lost.push("urls"); + if (lost.length > 0) { + this.warn(`Documentation ${lost.join("/")} on ${where} has no argtype node surface; ignored`); + } + } + + emit(expr: Expr, app?: AppMeta): string { + const raw = sinkDefaults(expr); + const lines: string[] = []; + + // argtype describes arguments only; the executable (argv[0]) belongs in + // frontmatter. Strip a leading command literal into `exe` so the body reads + // as arguments (the frontend re-prepends it on parse). The tool id stays the + // root label, which for a `wb_command ` tool differs from the exe. + let exe: string | undefined; + let root = raw; + if (raw.kind === "sequence" && raw.attrs.nodes[0]?.kind === "literal") { + exe = raw.attrs.nodes[0].attrs.str; + root = { kind: "sequence", attrs: { ...raw.attrs, nodes: raw.attrs.nodes.slice(1) } }; + if (raw.meta) root.meta = raw.meta; + } + + const frontmatter = this.emitFrontmatter( + app, + exe, + hasOutputs(root), + hasMediaTypes(root), + hasPaths(root), + ); + if (frontmatter) { + lines.push(frontmatter, ""); + } + + // The frontend wraps a non-sequence root in a synthetic single-child + // sequence (carrying only name + collected outputs). Emit the logical inner + // node as the definition body so the frontend re-wraps it identically; + // emitting the wrapper instead would re-attach the root doc to the wrapper. + const synthetic = syntheticWrap(root); + const defNode = synthetic ? synthetic.child : root; + const wrapperOutputs = synthetic ? synthetic.outputs : undefined; + + // The tool's title + description live in the root `///` block (from the + // root node's doc, or the app metadata for a converted descriptor) - unless + // the title convention would misread that block, in which case it round-trips + // as `.title()` / `.description()` chaining on the body instead. + const rootDoc = defNode.meta?.doc ?? app?.doc; + const rootChain = this.docNeedsChain(rootDoc); + if (!rootChain) lines.push(...this.docLines(rootDoc)); + // Only the node's own doc is checked here; an app doc's authors/urls are + // representable (frontmatter) and its literature/comment are warned there. + this.warnUnrepresentableDoc(defNode.meta?.doc, "the root node"); + + const rootName = this.labelFor(defNode.meta?.name || app?.id || "tool"); + let body = this.emitNode(defNode, 0); + if (rootChain) body += this.docChain(rootDoc); + if (wrapperOutputs?.length) body += this.emitOutputs(wrapperOutputs, 0); + lines.push(`${rootName}: ${body}`); + + return lines.join("\n") + "\n"; + } + + // -- Expression emission -- + + /** + * Emit a node's core expression plus its node-local chains (value + * constraints, default, join, count, media types) and any `.output(...)`. + * The first line is not indented (the caller places it after a label or pad); + * continuation lines are indented relative to `level`. + */ + private emitNode(expr: Expr, level: number): string { + let core: string; + switch (expr.kind) { + case "literal": + core = quote(expr.attrs.str); + break; + case "str": + core = "str" + this.defaultSuffix(expr, false); + break; + case "int": + case "float": { + core = expr.kind; + let chained = false; + const min = this.finiteNum(expr.attrs.minValue, "min"); + if (min !== undefined) { + core += `.min(${min})`; + chained = true; + } + const max = this.finiteNum(expr.attrs.maxValue, "max"); + if (max !== undefined) { + core += `.max(${max})`; + chained = true; + } + core += this.defaultSuffix(expr, chained); + break; + } + case "path": { + core = "path"; + const media = expr.attrs.mediaTypes ?? []; + for (const m of media) core += `.mediaType(${quote(m)})`; + // `.mutable()` / `.resolveParent()` are the `paths` extension. + if (expr.attrs.mutable) core += ".mutable()"; + if (expr.attrs.resolveParent) core += ".resolveParent()"; + const chained = media.length > 0 || !!expr.attrs.mutable || !!expr.attrs.resolveParent; + core += this.defaultSuffix(expr, chained); + break; + } + case "sequence": + core = this.emitCombinator("seq", expr.attrs.nodes, level, expr.attrs.join); + core += this.structuralDefaultSuffix(expr); + break; + case "optional": + core = this.emitOptional(expr, level) + this.structuralDefaultSuffix(expr); + break; + case "repeat": + core = this.emitRepeat(expr, level) + this.structuralDefaultSuffix(expr); + break; + case "alternative": + // argtype has no surface for an explicit union discriminator: the + // frontend re-derives an arm's `@type` tag from its label (name). That + // reproduces the discriminant whenever `variantTag` equals the emitted + // name (the common case), but a `variantTag` that was set to something + // else (to disambiguate a collapsed subcommand) would change on + // re-parse, so warn. + for (const arm of expr.attrs.alts) { + const tag = arm.meta?.variantTag; + if (tag !== undefined && tag !== arm.meta?.name) { + this.warn( + `Union arm discriminator '${tag}' has no argtype surface and will be re-derived ` + + `from the arm name '${arm.meta?.name ?? ""}' on re-parse`, + ); + } + } + core = + this.emitCombinator("alt", expr.attrs.alts, level) + this.structuralDefaultSuffix(expr); + break; + default: { + const _exhaustive: never = expr; + core = ""; + } + } + + if (expr.meta?.outputs?.length) { + core += this.emitOutputs(expr.meta.outputs, level); + } + return core; + } + + /** `String(n)` when `n` is finite; otherwise undefined with a warning, since + * `Infinity`/`NaN` have no argtype number literal (emitting them would produce + * an identifier that fails to re-parse). */ + private finiteNum(n: number | undefined, where: string): string | undefined { + if (n === undefined) return undefined; + if (!Number.isFinite(n)) { + this.warn(`Non-finite number (${String(n)}) on ${where} has no argtype literal; ignored`); + return undefined; + } + return num(n); + } + + /** `= value` on a bare terminal, else `.default(value)` once a chain started. */ + private defaultSuffix(expr: Expr, chained: boolean): string { + const dv = expr.meta?.defaultValue; + if (dv === undefined || typeof dv === "boolean") return ""; + let value: string; + if (typeof dv === "number") { + const n = this.finiteNum(dv, "default"); + if (n === undefined) return ""; + value = n; + } else { + value = quote(dv); + } + return chained ? `.default(${value})` : ` = ${value}`; + } + + /** + * A `.default(value)` chained onto a structural node (e.g. an enum + * `alternative`, or a wrapper whose default could not sink onto an inner + * terminal). Booleans have no argtype value literal: the only legitimate one + * is the `opt(literal)` flag-false convention, which the frontend regenerates, + * so it is silently dropped; any other boolean default is warned and dropped. + */ + private structuralDefaultSuffix(expr: Expr): string { + const dv = expr.meta?.defaultValue; + if (dv === undefined) return ""; + if (typeof dv === "boolean") { + const isFlagFalse = + expr.kind === "optional" && expr.attrs.node.kind === "literal" && dv === false; + if (!isFlagFalse) { + this.warn(`Boolean default on a ${expr.kind} cannot be expressed in argtype; ignored`); + } + return ""; + } + if (typeof dv === "number") { + const n = this.finiteNum(dv, "default"); + return n === undefined ? "" : `.default(${n})`; + } + return `.default(${quote(dv)})`; + } + + private emitOptional(expr: Optional, level: number): string { + const inner = expr.attrs.node; + if (inner.kind === "sequence" && !inner.meta && inner.attrs.join === undefined) { + return this.emitCombinator("opt", inner.attrs.nodes, level); + } + return this.emitCombinator("opt", [inner], level); + } + + private emitRepeat(expr: Repeat, level: number): string { + const node = expr.attrs.node; + let core: string; + if (node.kind === "sequence" && !node.meta && node.attrs.join === undefined) { + core = this.emitCombinator("rep", node.attrs.nodes, level); + } else { + core = this.emitCombinator("rep", [node], level); + } + if (expr.attrs.join !== undefined) { + core += `.join(${expr.attrs.join === "" ? "" : quote(expr.attrs.join)})`; + } + // `.count(n)` for an exact count; otherwise the composable `.countMin()` / + // `.countMax()` primitives, which express one-sided bounds too. + const { countMin, countMax } = expr.attrs; + if (countMin !== undefined && countMin === countMax) { + core += `.count(${countMin})`; + } else { + if (countMin !== undefined) core += `.countMin(${countMin})`; + if (countMax !== undefined) core += `.countMax(${countMax})`; + } + return core; + } + + private emitCombinator(keyword: string, children: Expr[], level: number, join?: string): string { + let core: string; + if (children.length === 0) { + core = `${keyword}()`; + } else { + const items = children.map((c) => this.emitDecorated(c, level + 1)); + // Stay on one line when no child carries a doc or spans lines and the + // result is short, so simple groups read like hand-written argtype. + const inline = `${keyword}(${items.join(", ")})`; + if (!items.some((i) => i.includes("\n")) && inline.length + level * INDENT.length <= 80) { + core = inline; + } else { + const padded = items.map((i) => pad(level + 1) + i); + core = `${keyword}(\n${padded.join(",\n")},\n${pad(level)})`; + } + } + if (join !== undefined) { + core += `.join(${join === "" ? "" : quote(join)})`; + } + return core; + } + + /** + * A list item: leading `///` doc lines, an optional `label:` prefix, then the + * node. The first line is unindented (the caller pads it); continuation lines + * are indented to `level`. + */ + private emitDecorated(expr: Expr, level: number): string { + const doc = expr.meta?.doc; + // A doc the title convention would misread is emitted as chaining on the node + // instead of a leading `///` block (see `docNeedsChain`). + const useChain = this.docNeedsChain(doc); + const parts: string[] = useChain ? [] : [...this.docLines(doc)]; + this.warnUnrepresentableDoc(doc, `node '${expr.meta?.name ?? expr.kind}'`); + // An empty name is dropped by the frontend on re-parse, so emit no label. + const label = expr.meta?.name ? `${this.labelFor(expr.meta.name)}: ` : ""; + let node = this.emitNode(expr, level); + if (useChain) node += this.docChain(doc); + parts.push(label + node); + return parts.join("\n" + pad(level)); + } + + // -- Outputs -- + + private emitOutputs(outputs: Output[], level: number): string { + const items = outputs.map((o) => { + // A `///` doc block precedes the output entry (title-convention split), + // unless that block would be misread - then it round-trips as `.title()` / + // `.description()` chaining on the template, as node docs do. + const useChain = this.docNeedsChain(o.doc); + const docs = useChain ? [] : this.docLines(o.doc).map((l) => pad(level + 1) + l); + let template = pad(level + 1) + this.emitOutputTemplate(o); + if (useChain) template += this.docChain(o.doc); + return [...docs, template].join("\n"); + }); + return `.output(\n${items.join(",\n")},\n${pad(level)})`; + } + + private emitOutputTemplate(output: Output): string { + if (output.mediaTypes?.length) { + this.warn( + `Output '${output.name ?? ""}' has media types, which have no argtype surface; ignored`, + ); + } + const label = output.name ? `${this.labelFor(output.name)}: ` : ""; + let body = ""; + for (const token of output.tokens) { + if (token.kind === "literal") { + // Escape template-significant characters so a literal `{`/`}`/backtick + // in the path round-trips (the frontend unescapes them). + body += token.value.replace(/[\\{}`]/g, (c) => `\\${c}`); + continue; + } + // Emit the ref name the same way as the target node's `label:` (bare + // identifier or quoted), so the `{ref}` and its `label:` stay in agreement. + let ref = identOrQuoted(token.target.name); + for (const ext of token.stripExtensions ?? []) ref += `.strip_suffix(${quote(ext)})`; + if (token.fallback !== undefined) ref += `.or(${quote(token.fallback)})`; + body += `{${ref}}`; + } + return `${label}\`${body}\``; + } + + // -- Frontmatter -- + + /** Quote a frontmatter scalar (double-quoted, backslash-escaped). The + * frontmatter parser unescapes symmetrically, so quotes/newlines round-trip. */ + private fmScalar(value: string): string { + return quote(value); + } + + private emitFrontmatter( + app: AppMeta | undefined, + exe: string | undefined, + outputs: boolean, + mediaTypes: boolean, + paths: boolean, + ): string | undefined { + const lines: string[] = []; + + // `exe` is the executable stripped from the command (argv[0]); emit it only + // when one was present, so the frontend's re-prepend stays symmetric. + if (exe !== undefined) lines.push(`exe: ${this.fmScalar(exe)}`); + + if (app) { + if (app.version) lines.push(`version: ${this.fmScalar(app.version)}`); + + const authors = app.doc?.authors ?? []; + if (authors.length > 0) { + lines.push("authors:"); + for (const a of authors) lines.push(` - ${this.fmScalar(a)}`); + } + + const urls = app.doc?.urls ?? []; + if (urls.length > 0) { + lines.push("urls:"); + for (const u of urls) lines.push(` - ${this.fmScalar(u)}`); + } + + const references = app.doc?.literature ?? []; + if (references.length > 0) { + lines.push("references:"); + for (const rf of references) lines.push(` - ${this.fmScalar(rf)}`); + } + + if (app.container) { + lines.push("container:"); + lines.push(` image: ${this.fmScalar(app.container.image)}`); + if (app.container.type) lines.push(` type: ${this.fmScalar(app.container.type)}`); + } + + if (app.stdout) this.emitStream(lines, "stdout", app.stdout); + if (app.stderr) this.emitStream(lines, "stderr", app.stderr); + + // app.doc.title is emitted in the root `///` block, not frontmatter. + if (app.doc?.comment) { + this.warn("AppMeta.doc.comment has no argtype surface; ignored"); + } + } + + const extensions: string[] = []; + if (outputs) extensions.push("outputs"); + if (mediaTypes) extensions.push("mediatypes"); + if (paths) extensions.push("paths"); + if (extensions.length > 0) { + lines.push("extensions:"); + for (const e of extensions) lines.push(` - ${e}`); + } + + if (lines.length === 0) return undefined; + return ["---", ...lines, "---"].join("\n"); + } + + private emitStream(lines: string[], key: string, stream: StreamOutput): void { + lines.push(`${key}:`); + lines.push(` name: ${this.fmScalar(stream.name)}`); + if (stream.doc?.description) + lines.push(` description: ${this.fmScalar(stream.doc.description)}`); + if (stream.doc?.title) { + this.warn(`${key} stream title has no argtype surface; ignored`); + } + } +} + +/** Emit argtype sugar-DSL source from an IR expression tree and optional `AppMeta`. */ +export function generateArgtype( + expr: Expr, + app?: AppMeta, +): { source: string; warnings: EmitWarning[] } { + const emitter = new ArgtypeEmitter(); + const source = emitter.emit(expr, app); + return { source, warnings: emitter.warnings }; +} diff --git a/packages/core/src/backend/argtype/index.ts b/packages/core/src/backend/argtype/index.ts new file mode 100644 index 0000000..7b5e20b --- /dev/null +++ b/packages/core/src/backend/argtype/index.ts @@ -0,0 +1,2 @@ +export { ArgtypeBackend } from "./argtype.js"; +export { generateArgtype } from "./emit.js"; diff --git a/packages/core/src/backend/argtype/roundtrip.test.ts b/packages/core/src/backend/argtype/roundtrip.test.ts new file mode 100644 index 0000000..a5912a3 --- /dev/null +++ b/packages/core/src/backend/argtype/roundtrip.test.ts @@ -0,0 +1,79 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { ArgtypeParser } from "../../frontend/argtype/parser-frontend.js"; +import { generateArgtype } from "./emit.js"; + +/** + * Structural round-trip: parse -> IR -> emit -> parse -> IR, asserting the two + * IRs are structurally equal. The emitter reproduces exactly the IR the + * frontend lowers to, so no normalization is needed: + * - `set`/`any` never appear (the frontend already lowered them away), so there + * is nothing for the emitter to recover. + * - Output declarations live on the root sequence (the frontend collects them + * there), so the emitter re-emits them on the root and they re-collect 1:1. + * - The `opt(literal)` flag-false default is regenerated by the frontend, so the + * emitter omits it and it reappears on the re-parse. + */ + +const parser = new ArgtypeParser(); + +function fixture(name: string): string { + return readFileSync(fileURLToPath(new URL(`./__fixtures__/${name}`, import.meta.url)), "utf8"); +} + +function frontendFixture(name: string): string { + return readFileSync( + fileURLToPath(new URL(`../../frontend/argtype/__fixtures__/${name}`, import.meta.url)), + "utf8", + ); +} + +function roundTrip(source: string) { + const first = parser.parse(source); + expect(first.errors).toEqual([]); + + const { source: emitted, warnings } = generateArgtype(first.expr, first.meta); + + const second = parser.parse(emitted); + expect(second.errors, `re-parse failed:\n${emitted}`).toEqual([]); + + return { first, second, emitted, warnings }; +} + +describe("argtype backend structural round-trip", () => { + it("round-trips bet.argtype (terminals, constraints, defaults, outputs)", () => { + const { first, second, warnings } = roundTrip(frontendFixture("bet.argtype")); + expect(warnings).toEqual([]); + expect(second.expr).toEqual(first.expr); + expect(second.meta).toEqual(first.meta); + }); + + it("round-trips a subcommand union (alt arms with labels and a positional default)", () => { + const { first, second, warnings } = roundTrip(fixture("subcommands.argtype")); + expect(warnings).toEqual([]); + // The command is ` `: the executable literal from frontmatter is + // prepended, followed by the subcommand alternative. + expect(first.expr.kind).toBe("sequence"); + const nodes = first.expr.kind === "sequence" ? first.expr.attrs.nodes : []; + expect(nodes[0]?.kind).toBe("literal"); + expect(nodes.some((n) => n.kind === "alternative")).toBe(true); + expect(second.expr).toEqual(first.expr); + expect(second.meta).toEqual(first.meta); + }); + + it("round-trips join micro-syntax and media types", () => { + const { first, second, warnings } = roundTrip(fixture("microsyntax.argtype")); + expect(warnings).toEqual([]); + expect(second.expr).toEqual(first.expr); + expect(second.meta).toEqual(first.meta); + }); + + it("is idempotent (emit -> parse -> emit produces identical text)", () => { + for (const name of ["subcommands.argtype", "microsyntax.argtype"]) { + const { emitted, second } = roundTrip(fixture(name)); + const reEmitted = generateArgtype(second.expr, second.meta).source; + expect(reEmitted).toBe(emitted); + } + }); +}); diff --git a/packages/core/src/backend/argtype/semantic-roundtrip.test.ts b/packages/core/src/backend/argtype/semantic-roundtrip.test.ts new file mode 100644 index 0000000..420eeb1 --- /dev/null +++ b/packages/core/src/backend/argtype/semantic-roundtrip.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; +import { defaultPipeline } from "../../ir/index.js"; +import { createContext } from "../../manifest/context.js"; +import { resolveOutputs, solve } from "../../solver/index.js"; +import { generatePython } from "../python/python.js"; +import { generateTypeScript } from "../typescript/typescript.js"; +import { BoutiquesParser } from "../../frontend/boutiques/parser.js"; +import { ArgtypeParser } from "../../frontend/argtype/parser-frontend.js"; +import type { ParseResult } from "../../frontend/frontend.js"; +import { generateArgtype } from "./emit.js"; + +/** + * Semantic round-trip: an existing-style Boutiques descriptor -> IR -> emit + * argtype -> parse back -> IR, then solve + codegen, asserting the generated + * TypeScript and Python match the direct Boutiques -> IR -> codegen path. + * + * The descriptor deliberately avoids the documented lossiness so the match is + * exact (no normalization): + * - No `name` fields (Boutiques `name` -> `doc.title`, which argtype has no + * surface for), only `description` (-> `///`). + * - Defaults only where the IR keeps them on a node argtype can decorate (a + * positional terminal, or an enum `alternative`), never hoisted onto a + * flag/optional wrapper. + * - No `mutable` / `resolve-parent` paths and no output docs. + */ + +const boutiques = new BoutiquesParser(); +const argtype = new ArgtypeParser(); + +const DESCRIPTOR = { + id: "resample", + "tool-version": "2.0", + author: "Test Author", + url: "https://example.org/resample", + "command-line": "resample [INPUT] [METHOD] [VERBOSE] [SCALE] [LABELS] [OUTPUT]", + inputs: [ + { id: "input", description: "Input volume", type: "File", "value-key": "[INPUT]" }, + { + id: "method", + description: "Resampling method", + type: "String", + "value-key": "[METHOD]", + "value-choices": ["linear", "nearest", "cubic"], + "default-value": "linear", + }, + { + id: "verbose", + description: "Verbose logging", + type: "Flag", + "command-line-flag": "-v", + "value-key": "[VERBOSE]", + }, + { + id: "scale", + description: "Scale factor", + type: "Number", + "value-key": "[SCALE]", + "command-line-flag": "--scale", + optional: true, + minimum: 0, + maximum: 10, + }, + { + id: "labels", + description: "Label values", + type: "Number", + integer: true, + list: true, + "value-key": "[LABELS]", + "command-line-flag": "--labels", + "list-separator": ",", + optional: true, + }, + { id: "output", description: "Output prefix", type: "String", "value-key": "[OUTPUT]" }, + ], + "output-files": [ + { + id: "out_file", + "path-template": "[OUTPUT].nii.gz", + "path-template-stripped-extensions": [".nii", ".nii.gz"], + }, + ], +}; + +function codegen(parsed: ParseResult) { + expect(parsed.errors).toEqual([]); + const optimized = defaultPipeline.apply(parsed.expr); + const solveResult = solve(optimized.expr); + const outputs = resolveOutputs(optimized.expr, solveResult); + expect(outputs.diagnostics.errors).toEqual([]); + const ctx = createContext(optimized.expr, solveResult, outputs, { + app: parsed.meta ?? { id: "tool" }, + package: { name: "pkg" }, + }); + return { ts: generateTypeScript(ctx), py: generatePython(ctx) }; +} + +describe("argtype backend semantic round-trip (Boutiques -> argtype -> codegen)", () => { + it("produces identical TypeScript and Python via the argtype detour", () => { + const direct = boutiques.parse(JSON.stringify(DESCRIPTOR)); + expect(direct.errors).toEqual([]); + + const { source, warnings } = generateArgtype(direct.expr, direct.meta); + expect(warnings).toEqual([]); + + const reparsed = argtype.parse(source); + expect(reparsed.errors, `re-parse failed:\n${source}`).toEqual([]); + + // The detour reconstructs the same IR (this is the strongest assertion; + // identical codegen follows from it). + expect(reparsed.expr).toEqual(direct.expr); + expect(reparsed.meta).toEqual(direct.meta); + + const directOut = codegen(direct); + const argtypeOut = codegen(reparsed); + + expect(argtypeOut.ts).toEqual(directOut.ts); + expect(argtypeOut.py).toEqual(directOut.py); + }); +}); diff --git a/packages/core/src/backend/index.ts b/packages/core/src/backend/index.ts index eb5af87..723f919 100644 --- a/packages/core/src/backend/index.ts +++ b/packages/core/src/backend/index.ts @@ -8,6 +8,7 @@ export type { EmitWarning, TypeMap, } from "./backend.js"; +export { ArgtypeBackend, generateArgtype } from "./argtype/index.js"; export { BoutiquesBackend, generateBoutiques } from "./boutiques/index.js"; export { CodeBuilder } from "./code-builder.js"; export type { FieldInfo } from "./collect-field-info.js"; diff --git a/packages/core/src/frontend/argtype/__fixtures__/bet.argtype b/packages/core/src/frontend/argtype/__fixtures__/bet.argtype new file mode 100644 index 0000000..6a384c0 --- /dev/null +++ b/packages/core/src/frontend/argtype/__fixtures__/bet.argtype @@ -0,0 +1,103 @@ +--- +exe: "bet" +version: "6.0.4" +authors: + - "FMRIB Analysis Group, University of Oxford" +urls: + - "https://fsl.fmrib.ox.ac.uk/fsl/fslwiki" +extensions: + - outputs +--- + +/// Automated brain extraction tool for FSL +bet: seq( + /// Input image (e.g. img.nii.gz). + infile: path, + + /// Output brain mask prefix (e.g. img_bet). + maskfile: str = "img_bet", + + set( + /// Fractional intensity threshold (0->1). Smaller values give larger + /// brain outline estimates. + opt("-f", fractional_intensity: float.min(0).max(1).default(0.5)), + + /// Vertical gradient in fractional intensity threshold (-1->1). + /// Positive values give larger brain outline at bottom, smaller at top. + opt("-g", vg_fractional_intensity: float.min(-1).max(1).default(0)), + + /// XYZ coordinates (voxels, not mm) of the center of gravity of the + /// initial mesh surface. + opt("-c", center_of_gravity: rep(float).count(3)), + + /// Head radius (mm, not voxels); initial surface sphere is set to half + /// of this. + opt("-r", head_radius: float), + + /// Generate brain surface outline overlaid onto original image. + overlay: opt("-o").output(overlay_file: `{maskfile}_overlay.nii.gz`), + + /// Generate binary brain mask. + binary_mask_flag: opt("-m").output(binary_mask: `{maskfile}_mask.nii.gz`), + + /// Generate rough skull image (not as clean as betsurf). + approx_skull: opt("-s").output( + approx_skull_img: `{maskfile}_skull.nii.gz`, + skull_mask: `{maskfile}_skull_mask.nii.gz`, + ), + + /// Don't generate segmented brain image output. + no_seg_output: opt("-n"), + + /// Generate brain surface as mesh in .vtk format. + vtk_mesh: opt("-e").output(output_vtk_mesh: `{maskfile}_mesh.vtk`), + + /// Apply thresholding to segmented brain image and mask. + thresholding: opt("-t"), + + /// More robust brain center estimation by iterating BET with a + /// changing center-of-gravity. + robust_iters: opt("-R"), + + /// Cleanup residual eye and optic nerve voxels (useful before + /// SIENA/SIENAX). + residual_optic_cleanup: opt("-S"), + + /// Reduce image bias and residual neck voxels (useful before + /// SIENA/SIENAX). + reduce_bias: opt("-B"), + + /// Pad end slices in both directions; useful when only a few slices + /// are present in the Z direction. + slice_padding: opt("-Z"), + + /// Determine brain mask from the first volume of a 4D dataset and + /// apply it to the whole set. Intended for FMRI; the -f threshold + /// is reduced to 0.3. + whole_set_mask: opt("-F"), + + /// Run bet2 + betsurf so skull and scalp surfaces are also produced + /// (requires registration to standard space). + additional_surfaces: opt("-A").output( + out_inskull_mask: `{maskfile}_inskull_mask.nii.gz`, + out_inskull_mesh: `{maskfile}_inskull_mesh.nii.gz`, + out_inskull_off: `{maskfile}_inskull_mesh.off`, + out_outskin_mask: `{maskfile}_outskin_mask.nii.gz`, + out_outskin_mesh: `{maskfile}_outskin_mesh.nii.gz`, + out_outskin_off: `{maskfile}_outskin_mesh.off`, + out_outskull_mask: `{maskfile}_outskull_mask.nii.gz`, + out_outskull_mesh: `{maskfile}_outskull_mesh.nii.gz`, + out_outskull_off: `{maskfile}_outskull_mesh.off`, + ), + + /// Same as -A, but also takes a T2 image as input to refine the + /// skull/scalp surfaces. + opt("-A2", additional_surfaces_t2: path), + + /// Switch on diagnostic messages. + verbose: opt("-v"), + + /// Don't delete temporary intermediate images. + debug: opt("-d"), + ), +).output(outfile: `{maskfile}.nii.gz`) diff --git a/packages/core/src/frontend/argtype/ast.ts b/packages/core/src/frontend/argtype/ast.ts new file mode 100644 index 0000000..05d8eab --- /dev/null +++ b/packages/core/src/frontend/argtype/ast.ts @@ -0,0 +1,100 @@ +/** + * AST for the argtype sugar DSL. Mirrors the grammar in the argtype spec + * (combinators, terminals, literals, naming, aliases, method chains) and is the + * intermediate the parser produces before `lower.ts` translates it to Styx IR. + * + * Decorations (`name`, `doc`, `default`, value constraints, `.join`/`.count`, + * outputs, media types) attach to any node, so they live as optional fields on + * the single `AstNode` shape rather than as wrapper nodes - lowering folds them + * onto the corresponding `NodeMeta` / terminal attrs. + */ + +/** A combinator keyword. `set`/`any` have no direct IR node (see `lower.ts`). */ +export type Combinator = "seq" | "set" | "opt" | "rep" | "alt" | "any"; + +/** A terminal keyword. */ +export type Terminal = "int" | "float" | "str" | "path"; + +export interface AstNode { + kind: "literal" | "terminal" | "comb" | "ref"; + + /** kind === "literal": the fixed token string. */ + value?: string; + /** kind === "terminal": which terminal. */ + terminal?: Terminal; + /** kind === "comb": which combinator + its children. */ + op?: Combinator; + children?: AstNode[]; + /** kind === "ref": an alias reference (resolved by substitution). */ + refName?: string; + + // -- Decorations (any node) -- + + /** `label:` sugar / `.name("...")`. */ + name?: string; + /** Title: a `# ` heading in a `///` block, or `.title("...")`. */ + title?: string; + /** Description: a `///` block (minus the title), or `.description("...")`. */ + description?: string; + /** `= value` sugar / `.default(...)` (terminal only). */ + default?: string | number; + /** `.min(n)` (int/float only). */ + min?: number; + /** `.max(n)` (int/float only). */ + max?: number; + /** `.join(sep?)` - present means joined; separator defaults to "". */ + join?: string; + /** `.count(n)` (= `.countMin(n).countMax(n)`) / `.countMin(n)` / `.countMax(n)` + * (rep only). */ + countMin?: number; + countMax?: number; + /** `.mediaType(mime)` (path only; `mediatypes` extension). */ + mediaTypes?: string[]; + /** `.mutable()` (path only; `paths` extension). */ + mutable?: boolean; + /** `.resolveParent()` (path only; `paths` extension). */ + resolveParent?: boolean; + /** `.output(...)` (`outputs` extension). */ + outputs?: AstOutput[]; +} + +/** One output template declared via `.output(...)`. */ +export interface AstOutput { + /** `label:` on the template / `.name(...)`. */ + name?: string; + /** Title of the produced file (from a `# ` heading in the entry's `///`). */ + title?: string; + /** Description of the produced file (from the entry's `///`). */ + description?: string; + /** Output-level `.or(fallback)` on the template. */ + fallback?: string; + tokens: AstOutputToken[]; +} + +/** A piece of an output path template: literal text or a `{ref}` interpolation. */ +export type AstOutputToken = + | { kind: "literal"; value: string } + | { + kind: "ref"; + /** Target node name; undefined for a self-reference (`{}`). */ + name?: string; + stripSuffix?: string[]; + stripPrefix?: string[]; + basename?: boolean; + or?: string; + }; + +/** A top-level alias definition (`Name = expr`). */ +export interface AstAlias { + name: string; + expr: AstNode; +} + +/** The result of parsing a full argtype document. */ +export interface AstDocument { + frontmatter?: Record; + aliases: AstAlias[]; + /** The single root definition (`name: expr`). */ + rootName: string; + root: AstNode; +} diff --git a/packages/core/src/frontend/argtype/doc.ts b/packages/core/src/frontend/argtype/doc.ts new file mode 100644 index 0000000..74c67ff --- /dev/null +++ b/packages/core/src/frontend/argtype/doc.ts @@ -0,0 +1,33 @@ +/** + * Doc-comment title/description handling. + * + * The `# Title` heading convention is *sugar*: it applies only when splitting a + * `///` block (`splitDocText`). The chaining methods `.title()` / `.description()` + * set the title / description directly and are not re-parsed for headings. + */ + +export interface DocParts { + title?: string; + description?: string; +} + +/** + * Split a `///` doc block into title + description. A leading Markdown H1 + * (`# Title` on the first line) is the title; everything after it is the + * description. Without a leading `# `, the whole block is the description (no + * title), regardless of paragraph breaks. + */ +export function splitDocText(raw: string): DocParts { + const newline = raw.indexOf("\n"); + const firstLine = (newline === -1 ? raw : raw.slice(0, newline)).trim(); + if (firstLine.startsWith("# ")) { + const title = firstLine.slice(2).trim(); + const description = (newline === -1 ? "" : raw.slice(newline + 1)).trim(); + return { + ...(title && { title }), + ...(description && { description }), + }; + } + const description = raw.trim(); + return description ? { description } : {}; +} diff --git a/packages/core/src/frontend/argtype/end-to-end.test.ts b/packages/core/src/frontend/argtype/end-to-end.test.ts new file mode 100644 index 0000000..69bb5a3 --- /dev/null +++ b/packages/core/src/frontend/argtype/end-to-end.test.ts @@ -0,0 +1,80 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { defaultPipeline } from "../../ir/index.js"; +import { createContext } from "../../manifest/context.js"; +import { resolveOutputs, solve } from "../../solver/index.js"; +import { generatePython } from "../../backend/python/python.js"; +import { generateTypeScript } from "../../backend/typescript/typescript.js"; +import { generateSchema } from "../../backend/schema/jsonschema.js"; +import { ArgtypeParser } from "./parser-frontend.js"; + +const parser = new ArgtypeParser(); + +function fixture(name: string): string { + return readFileSync(fileURLToPath(new URL(`./__fixtures__/${name}`, import.meta.url)), "utf8"); +} + +function run(source: string) { + const parseResult = parser.parse(source); + expect(parseResult.errors).toEqual([]); + const optimized = defaultPipeline.apply(parseResult.expr); + const solveResult = solve(optimized.expr); + const outputs = resolveOutputs(optimized.expr, solveResult); + expect(outputs.diagnostics.errors).toEqual([]); + const ctx = createContext(optimized.expr, solveResult, outputs, { + app: parseResult.meta ?? { id: "tool" }, + package: { name: "pkg" }, + }); + return { parseResult, ctx, outputs }; +} + +describe("argtype end-to-end: bet", () => { + it("compiles bet.argtype through to TypeScript and Python", () => { + const { parseResult, ctx, outputs } = run(fixture("bet.argtype")); + + // Frontmatter -> AppMeta. + expect(parseResult.meta?.id).toBe("bet"); + expect(parseResult.meta?.version).toBe("6.0.4"); + + const ts = generateTypeScript(ctx); + const py = generatePython(ctx); + const schema = generateSchema(ctx) as Record; + + // Core parameters survive into the typed interface. + for (const field of ["infile", "maskfile", "fractional_intensity", "center_of_gravity"]) { + expect(ts).toContain(field); + expect(py).toContain(field); + } + + // The output-file declarations are collected and resolved. + const allOutputs = outputs.scopes.flatMap((s) => s.outputs); + const outNames = allOutputs.map((o) => o.name); + expect(outNames).toContain("outfile"); + expect(outNames).toContain("binary_mask"); + + // The `{maskfile}_mask.nii.gz` template resolves to a ref to the maskfile + // binding plus the literal suffix - not just present-by-name. + const binaryMask = allOutputs.find((o) => o.name === "binary_mask")!; + expect(binaryMask.tokens).toHaveLength(2); + expect(binaryMask.tokens[0]?.kind).toBe("ref"); + expect(binaryMask.tokens[1]).toEqual({ kind: "literal", value: "_mask.nii.gz" }); + if (binaryMask.tokens[0]?.kind === "ref") { + expect(ctx.bindings.get(binaryMask.tokens[0].binding)?.name).toBe("maskfile"); + } + + // Sanity: generated code is non-trivial and references the executable. + expect(ts.length).toBeGreaterThan(200); + expect(ts).toContain("bet"); + expect(schema).toBeTruthy(); + }); + + it("models the 3-coordinate center as a fixed-count list", () => { + const { ctx } = run(`tool: set( + opt("--center", center: rep(float).count(3)), + input: path, + )`); + const ts = generateTypeScript(ctx); + expect(ts).toContain("center"); + }); +}); diff --git a/packages/core/src/frontend/argtype/frontmatter.ts b/packages/core/src/frontend/argtype/frontmatter.ts new file mode 100644 index 0000000..a01a177 --- /dev/null +++ b/packages/core/src/frontend/argtype/frontmatter.ts @@ -0,0 +1,193 @@ +/** + * Frontmatter handling for argtype documents. + * + * A document may begin with a `---` fenced block of YAML-ish metadata (`exe`, + * `version`, `extensions`, `container`, `authors`, `stdout`, ...). Rather than + * pull in a full YAML dependency, this parses the small subset the format uses: + * scalar `key: value`, block sequences (`- item`), and one level of nested + * mappings. That covers every key the lowering step consumes; anything more + * exotic is read as a raw string and ignored downstream. + */ + +export interface SplitResult { + frontmatter?: Record; + /** The document body. Frontmatter lines are blanked (not removed) so token + * line numbers still line up with the original source. */ + body: string; + errors: string[]; +} + +export function splitFrontmatter(source: string): SplitResult { + // Tolerate a leading BOM / blank lines before the opening fence. + const lines = source.split("\n"); + let start = 0; + while (start < lines.length && lines[start]!.trim() === "") start++; + + if (lines[start]?.trim() !== "---") { + return { body: source, errors: [] }; + } + + // Find the closing fence. + let end = -1; + for (let i = start + 1; i < lines.length; i++) { + if (lines[i]!.trim() === "---") { + end = i; + break; + } + } + if (end === -1) { + return { body: source, errors: ["Unterminated frontmatter block (missing closing '---')"] }; + } + + const fmLines = lines.slice(start + 1, end); + const { value, errors } = parseYamlish(fmLines); + + // Blank out the frontmatter region (fences + content) to preserve line numbers. + const blanked = lines.map((line, i) => (i >= start && i <= end ? "" : line)); + return { frontmatter: value, body: blanked.join("\n"), errors }; +} + +interface Line { + indent: number; + content: string; +} + +function tokenizeLines(raw: string[]): Line[] { + const out: Line[] = []; + for (const line of raw) { + const noComment = stripComment(line); + if (noComment.trim() === "") continue; + const indent = noComment.length - noComment.trimStart().length; + out.push({ indent, content: noComment.trim() }); + } + return out; +} + +/** Strip a trailing `#` comment that is not inside quotes. Following YAML, a + * `#` only starts a comment at the start of the line or after whitespace, so an + * unquoted value containing `#` (e.g. a URL fragment `https://x/#frag`) is kept + * intact. */ +function stripComment(line: string): string { + let inQuote: string | null = null; + for (let i = 0; i < line.length; i++) { + const ch = line[i]!; + // A backslash escape inside a double-quoted value hides the next char (so + // `\"` does not close the string). + if (inQuote === '"' && ch === "\\") { + i++; + continue; + } + if (inQuote) { + if (ch === inQuote) inQuote = null; + } else if (ch === '"' || ch === "'") { + inQuote = ch; + } else if (ch === "#" && (i === 0 || line[i - 1] === " " || line[i - 1] === "\t")) { + return line.slice(0, i); + } + } + return line; +} + +function parseYamlish(raw: string[]): { value: Record; errors: string[] } { + const lines = tokenizeLines(raw); + const errors: string[] = []; + let pos = 0; + + function parseBlock(minIndent: number): Record | unknown[] { + // Sequence block? + if ( + pos < lines.length && + lines[pos]!.indent >= minIndent && + lines[pos]!.content.startsWith("- ") + ) { + const seq: unknown[] = []; + const indent = lines[pos]!.indent; + while ( + pos < lines.length && + lines[pos]!.indent === indent && + lines[pos]!.content.startsWith("- ") + ) { + seq.push(parseScalar(lines[pos]!.content.slice(2).trim())); + pos++; + } + return seq; + } + + // Mapping block. + const map: Record = {}; + if (pos >= lines.length) return map; + const indent = lines[pos]!.indent; + while ( + pos < lines.length && + lines[pos]!.indent === indent && + !lines[pos]!.content.startsWith("- ") + ) { + const { content } = lines[pos]!; + const colon = findColon(content); + if (colon === -1) { + errors.push(`Malformed frontmatter line: '${content}'`); + pos++; + continue; + } + const key = content.slice(0, colon).trim(); + const valueText = content.slice(colon + 1).trim(); + pos++; + if (valueText !== "") { + map[key] = parseScalar(valueText); + } else if (pos < lines.length && lines[pos]!.indent > indent) { + map[key] = parseBlock(lines[pos]!.indent); + } else { + map[key] = null; + } + } + return map; + } + + const value = parseBlock(0); + return { value: isRecord(value) ? value : { _root: value }, errors }; +} + +function findColon(s: string): number { + let inQuote: string | null = null; + for (let i = 0; i < s.length; i++) { + const ch = s[i]!; + if (inQuote === '"' && ch === "\\") { + i++; + continue; + } + if (inQuote) { + if (ch === inQuote) inQuote = null; + } else if (ch === '"' || ch === "'") { + inQuote = ch; + } else if (ch === ":") { + return i; + } + } + return -1; +} + +/** Reverse the backslash escapes an emitter applies to a quoted scalar + * (`\n`, `\t`, `\r`, `\"`, `\\`), so values with newlines/quotes round-trip. */ +function unescapeScalar(s: string): string { + return s.replace(/\\(.)/g, (_, c) => + c === "n" ? "\n" : c === "t" ? "\t" : c === "r" ? "\r" : c, + ); +} + +function parseScalar(text: string): unknown { + if (text.startsWith('"') && text.endsWith('"') && text.length >= 2) { + return unescapeScalar(text.slice(1, -1)); + } + if (text.startsWith("'") && text.endsWith("'") && text.length >= 2) { + return text.slice(1, -1); + } + if (text === "true") return true; + if (text === "false") return false; + if (text === "null" || text === "~") return null; + if (/^-?\d+(\.\d+)?$/.test(text)) return Number(text); + return text; +} + +function isRecord(x: unknown): x is Record { + return typeof x === "object" && x !== null && !Array.isArray(x); +} diff --git a/packages/core/src/frontend/argtype/index.ts b/packages/core/src/frontend/argtype/index.ts new file mode 100644 index 0000000..7e9a125 --- /dev/null +++ b/packages/core/src/frontend/argtype/index.ts @@ -0,0 +1 @@ +export { ArgtypeParser } from "./parser-frontend.js"; diff --git a/packages/core/src/frontend/argtype/lexer.ts b/packages/core/src/frontend/argtype/lexer.ts new file mode 100644 index 0000000..6e9e3e4 --- /dev/null +++ b/packages/core/src/frontend/argtype/lexer.ts @@ -0,0 +1,244 @@ +/** + * Tokenizer for the argtype sugar DSL. + * + * Notable points: + * - `///` doc comments are significant tokens (they attach to the next node); + * `//` line comments are discarded. + * - Template literals (`` `{ref}.png` ``) are captured whole as a single token; + * their internal `{...}` structure is parsed separately by `template.ts`. + * - Combinator/terminal words (`seq`, `int`, ...) are plain identifiers; the + * parser classifies them, so they are not reserved at the lexer level. + */ + +export type TokenKind = + | "ident" + | "string" + | "number" + | "template" + | "doc" + | "colon" + | "eq" + | "pipe" + | "dot" + | "comma" + | "lparen" + | "rparen" + | "eof"; + +export interface Token { + kind: TokenKind; + /** Raw text: identifier name, unquoted string body, number text, doc text, or template body. */ + value: string; + line: number; + column: number; +} + +export interface LexError { + message: string; + line: number; + column: number; +} + +export interface LexResult { + tokens: Token[]; + errors: LexError[]; +} + +const PUNCT: Record = { + ":": "colon", + "=": "eq", + "|": "pipe", + ".": "dot", + ",": "comma", + "(": "lparen", + ")": "rparen", +}; + +function isIdentStart(ch: string): boolean { + return /[A-Za-z_]/.test(ch); +} + +function isIdentPart(ch: string): boolean { + return /[A-Za-z0-9_]/.test(ch); +} + +function isDigit(ch: string): boolean { + return ch >= "0" && ch <= "9"; +} + +export function lex(source: string): LexResult { + const tokens: Token[] = []; + const errors: LexError[] = []; + let i = 0; + let line = 1; + let col = 1; + + const peek = (o = 0): string => source[i + o] ?? ""; + const advance = (): string => { + const ch = source[i++]!; + if (ch === "\n") { + line++; + col = 1; + } else { + col++; + } + return ch; + }; + const push = (kind: TokenKind, value: string, l: number, c: number): void => { + tokens.push({ kind, value, line: l, column: c }); + }; + + while (i < source.length) { + const ch = peek(); + const startLine = line; + const startCol = col; + + // Whitespace (incl. CR for Windows line endings). + if (ch === " " || ch === "\t" || ch === "\r" || ch === "\n") { + advance(); + continue; + } + + // Comments: `///` doc (kept) vs `//` line (dropped). + if (ch === "/" && peek(1) === "/") { + const isDoc = peek(2) === "/"; + // Consume the slashes. + advance(); + advance(); + if (isDoc) advance(); + let text = ""; + while (i < source.length && peek() !== "\n") text += advance(); + if (isDoc) push("doc", text.trim(), startLine, startCol); + continue; + } + + // Double-quoted string literal. + if (ch === '"') { + advance(); + let str = ""; + let closed = false; + while (i < source.length) { + const c = advance(); + if (c === "\\") { + const esc = i < source.length ? advance() : ""; + str += unescape(esc); + } else if (c === '"') { + closed = true; + break; + } else if (c === "\n") { + break; + } else { + str += c; + } + } + if (!closed) + errors.push({ message: "Unterminated string literal", line: startLine, column: startCol }); + push("string", str, startLine, startCol); + continue; + } + + // Template literal: capture the raw body between backticks. A backslash + // escapes the next character (kept raw here, unescaped by `template.ts`), so + // an escaped backtick does not close the template. + if (ch === "`") { + advance(); + let body = ""; + let closed = false; + while (i < source.length) { + const c = advance(); + if (c === "\\") { + body += c; + if (i < source.length) body += advance(); + continue; + } + if (c === "`") { + closed = true; + break; + } + body += c; + } + if (!closed) + errors.push({ + message: "Unterminated template literal", + line: startLine, + column: startCol, + }); + push("template", body, startLine, startCol); + continue; + } + + // Number: optional leading `-`, digits, optional fractional part, optional + // exponent (e.g. `1e-05`, `2.2e-308` - tool value bounds use these). + if (isDigit(ch) || (ch === "-" && isDigit(peek(1)))) { + let num = advance(); // first char (digit or '-') + while (i < source.length && isDigit(peek())) num += advance(); + if (peek() === "." && isDigit(peek(1))) { + num += advance(); // '.' + while (i < source.length && isDigit(peek())) num += advance(); + } + // Exponent: e/E, optional sign, digits. + if (peek() === "e" || peek() === "E") { + const signed = peek(1) === "+" || peek(1) === "-"; + if (isDigit(peek(1)) || (signed && isDigit(peek(2)))) { + num += advance(); // e/E + if (peek() === "+" || peek() === "-") num += advance(); + while (i < source.length && isDigit(peek())) num += advance(); + } else { + // `e`/`E` right after the digits but no exponent value follows: a + // malformed number (digits never abut a bare identifier in argtype), + // so flag it rather than silently emitting `1` + ident `e`. + num += advance(); // consume the stray e/E + errors.push({ + message: `Malformed number '${num}': exponent has no digits`, + line: startLine, + column: startCol, + }); + } + } + push("number", num, startLine, startCol); + continue; + } + + // Identifier / keyword. + if (isIdentStart(ch)) { + let id = advance(); + while (i < source.length && isIdentPart(peek())) id += advance(); + push("ident", id, startLine, startCol); + continue; + } + + // Punctuation. + const punct = PUNCT[ch]; + if (punct) { + advance(); + push(punct, ch, startLine, startCol); + continue; + } + + errors.push({ message: `Unexpected character '${ch}'`, line: startLine, column: startCol }); + advance(); + } + + push("eof", "", line, col); + return { tokens, errors }; +} + +/** Minimal escape handling inside double-quoted strings. */ +function unescape(esc: string): string { + switch (esc) { + case "n": + return "\n"; + case "t": + return "\t"; + case "r": + return "\r"; + case '"': + return '"'; + case "\\": + return "\\"; + case "": + return ""; + default: + return esc; + } +} diff --git a/packages/core/src/frontend/argtype/lower.ts b/packages/core/src/frontend/argtype/lower.ts new file mode 100644 index 0000000..224dae8 --- /dev/null +++ b/packages/core/src/frontend/argtype/lower.ts @@ -0,0 +1,496 @@ +/** + * Lower an `AstDocument` to Styx IR (`Expr` + `AppMeta`). + * + * Mapping highlights: + * - Combinators map 1:1 except `set` -> `sequence` (order-not-meaningful is not + * modeled in the IR) and `any` -> its first branch (the spec's "emit branch 0" + * rule; the interchangeable alternatives are dropped with a warning). + * - Aliases are resolved by substitution with cycle detection. + * - `.output(...)` declarations attach to the nearest enclosing sequence scope, + * so an output nested in a repeat / subcommand keeps its list / struct shape + * (per-output gating is recovered downstream from each ref binding's gate). + */ + +import { alt, float, int, lit, opt, path as pathTerm, rep, seq, str } from "../../ir/builders.js"; +import { nodeRef } from "../../ir/meta.js"; +import type { AppMeta, NodeMeta, Output, OutputToken, StreamOutput } from "../../ir/meta.js"; +import type { Documentation } from "../../ir/types.js"; +import type { Expr, Sequence } from "../../ir/node.js"; +import type { AstAlias, AstDocument, AstNode, AstOutput } from "./ast.js"; + +export interface LowerResult { + meta?: AppMeta; + expr: Sequence; + errors: string[]; + warnings: string[]; +} + +/** Build IR `Documentation` from an AST node's already-split title/description. */ +function docFrom(node: { title?: string; description?: string }): Documentation | undefined { + const doc: Documentation = { + ...(node.title && { title: node.title }), + ...(node.description && { description: node.description }), + }; + return Object.keys(doc).length > 0 ? doc : undefined; +} + +class Lowerer { + readonly errors: string[] = []; + readonly warnings: string[] = []; + private aliases = new Map(); + + constructor(aliases: AstAlias[]) { + for (const a of aliases) { + if (this.aliases.has(a.name)) + this.warnings.push(`Duplicate alias '${a.name}'; last definition wins`); + this.aliases.set(a.name, a.expr); + } + } + + lower(doc: AstDocument): LowerResult { + // Each output attaches to its nearest enclosing sequence scope (via the + // `sink`), preserving nesting so a per-repeat / per-subcommand output keeps + // its list/struct shape. `rootSink` catches outputs declared directly on a + // non-sequence root (a sequence root manages its own outputs). + const rootSink: Output[] = []; + const expr = this.lowerNode(doc.root, new Set(), undefined, rootSink); + + const root: Sequence = expr.kind === "sequence" ? expr : seq(expr); + if (root !== expr) { + root.meta = { ...(doc.root.name && { name: doc.root.name }) }; + } + if (rootSink.length > 0) { + root.meta = { ...root.meta, outputs: [...(root.meta?.outputs ?? []), ...rootSink] }; + } + + return { expr: root, errors: this.errors, warnings: this.warnings }; + } + + /** + * @param expanding - alias names currently being expanded (cycle guard). + * @param selfName - nearest enclosing named node, for `{}` self-references. + * @param sink - outputs array of the nearest enclosing sequence; a `.output()` + * on this node attaches here (seq/set nodes instead own their outputs). + */ + private lowerNode( + node: AstNode, + expanding: Set, + selfName: string | undefined, + sink: Output[], + ): Expr { + // Alias reference: substitute then lower. + if (node.kind === "ref") { + return this.lowerRef(node, expanding, selfName, sink); + } + + const name = node.name ?? selfName; + + // `.join` collapses a node's subtree into one argv element. It is carried on + // sequence/repeat in the IR directly, and on `opt` by pushing it onto the + // wrapped content (`lowerComb`). On other nodes it has no meaning, so warn. + const joinable = + node.kind === "comb" && + (node.op === "seq" || node.op === "set" || node.op === "rep" || node.op === "opt"); + if (node.join !== undefined && !joinable) { + this.warnings.push("`.join()` is only supported on seq/set/rep/opt; ignored here"); + } + + // A `.output()` on a non-sequence node attaches to the enclosing sequence + // scope (`sink`); seq/set own their outputs (handled in `lowerComb`). + const isSeqSet = node.kind === "comb" && (node.op === "seq" || node.op === "set"); + if (node.outputs?.length && !isSeqSet) { + for (const o of node.outputs) sink.push(this.toOutput(o, name)); + } + + switch (node.kind) { + case "literal": { + const e = lit(node.value ?? ""); + this.applyMeta(e, node); + return e; + } + case "terminal": + return this.lowerTerminal(node); + case "comb": + return this.lowerComb(node, expanding, name, sink); + default: { + this.errors.push(`Unknown AST node kind '${(node as AstNode).kind}'`); + return seq(); + } + } + } + + private lowerRef( + node: AstNode, + expanding: Set, + selfName: string | undefined, + sink: Output[], + ): Expr { + const target = node.refName!; + const aliasExpr = this.aliases.get(target); + if (!aliasExpr) { + this.errors.push(`Unknown alias '${target}'`); + return seq(); + } + if (expanding.has(target)) { + this.errors.push(`Recursive alias '${target}' is not allowed`); + return seq(); + } + const clone = structuredClone(aliasExpr); + // Overlay use-site decorations onto the inlined alias. The use site wins for + // each scalar decoration it specifies; outputs and media types accumulate. + // Without this, `size: Dimension.join(",")` would silently drop the join. + clone.name = node.name ?? clone.name; + clone.title = node.title ?? clone.title; + clone.description = node.description ?? clone.description; + if (node.default !== undefined) clone.default = node.default; + if (node.min !== undefined) clone.min = node.min; + if (node.max !== undefined) clone.max = node.max; + if (node.join !== undefined) clone.join = node.join; + if (node.countMin !== undefined) clone.countMin = node.countMin; + if (node.countMax !== undefined) clone.countMax = node.countMax; + if (node.mediaTypes?.length) + clone.mediaTypes = [...(clone.mediaTypes ?? []), ...node.mediaTypes]; + if (node.mutable) clone.mutable = true; + if (node.resolveParent) clone.resolveParent = true; + if (node.outputs?.length) clone.outputs = [...(clone.outputs ?? []), ...node.outputs]; + + const next = new Set(expanding); + next.add(target); + return this.lowerNode(clone, next, selfName, sink); + } + + private lowerTerminal(node: AstNode): Expr { + switch (node.terminal) { + case "int": { + const e = int(); + this.checkBounds(node.min, node.max, "value", "min", "max"); + if (node.min !== undefined) e.attrs.minValue = node.min; + if (node.max !== undefined) e.attrs.maxValue = node.max; + this.applyMeta(e, node); + return e; + } + case "float": { + const e = float(); + this.checkBounds(node.min, node.max, "value", "min", "max"); + if (node.min !== undefined) e.attrs.minValue = node.min; + if (node.max !== undefined) e.attrs.maxValue = node.max; + this.applyMeta(e, node); + return e; + } + case "str": { + const e = str(); + if (node.mediaTypes?.length) { + // The IR carries media types on `path` only; a `str.mediaType(...)` has + // nowhere to live, so warn rather than dropping it silently. + this.warnings.push("`.mediaType()` on a `str` is not represented in the IR; ignored"); + } + this.applyMeta(e, node); + return e; + } + case "path": { + const e = pathTerm(); + if (node.mediaTypes?.length) e.attrs.mediaTypes = node.mediaTypes; + if (node.mutable) e.attrs.mutable = true; + if (node.resolveParent) e.attrs.resolveParent = true; + this.applyMeta(e, node); + return e; + } + default: { + this.errors.push(`Unknown terminal '${String(node.terminal)}'`); + return str(); + } + } + } + + private lowerComb( + node: AstNode, + expanding: Set, + name: string | undefined, + sink: Output[], + ): Expr { + const children = node.children ?? []; + const lowerChildren = (s: Output[]): Expr[] => + children.map((c) => this.lowerNode(c, expanding, name, s)); + + switch (node.op) { + case "seq": + case "set": { + // `set` is lowered to a sequence: the IR does not model "order not + // meaningful". A sequence is an output scope, so its own `.output()`s + // and any outputs its children declare attach here (not the parent). + const selfOutputs: Output[] = []; + for (const o of node.outputs ?? []) selfOutputs.push(this.toOutput(o, name)); + const lowered = lowerChildren(selfOutputs); + this.dedupeSiblingNames(lowered); + const e = seq(...lowered); + if (node.join !== undefined) e.attrs.join = node.join; + this.applyMeta(e, node); + if (selfOutputs.length > 0) { + e.meta = { ...e.meta, outputs: [...(e.meta?.outputs ?? []), ...selfOutputs] }; + } + return e; + } + case "alt": { + const e = alt(...lowerChildren(sink)); + this.applyMeta(e, node); + return e; + } + case "any": { + // Emit the first branch only. The `any` branches are interchangeable + // spellings of one token (`--output` / `-output` / `-o`), and the spec + // makes branch 0 authoritative for builders - so for a generator this + // is lossless and expected, not a degradation worth warning about. + // (Only a parser/validator would need to accept the other spellings.) + if (children.length === 0) { + this.errors.push("`any(...)` requires at least one branch"); + return seq(); + } + const e = this.lowerNode(children[0]!, expanding, name, sink); + // Overlay the any's own name/doc onto the emitted branch. + if (node.name) e.meta = { ...e.meta, name: node.name }; + { + const d = docFrom(node); + if (d) e.meta = { ...e.meta, doc: d }; + } + return e; + } + case "opt": { + const inner = this.wrapChildren(children, expanding, name, sink); + // `.join()` on the opt collapses its content into one argv element; push + // it onto the inner sequence/repeat (a single terminal is already one + // element, so a join there is a harmless no-op). + if (node.join !== undefined && (inner.kind === "sequence" || inner.kind === "repeat")) { + inner.attrs.join = node.join; + } + const e = opt(inner); + this.applyMeta(e, node); + // A bare-literal flag resolves to a bool; give it a false default to + // match the other frontends' flag convention. + if (inner.kind === "literal") e.meta = { ...e.meta, defaultValue: false }; + return e; + } + case "rep": { + const inner = this.wrapChildren(children, expanding, name, sink); + const e = rep(inner); + if (node.join !== undefined) e.attrs.join = node.join; + this.checkBounds(node.countMin, node.countMax, "repetition count", "countMin", "countMax"); + if (node.countMin !== undefined) e.attrs.countMin = node.countMin; + if (node.countMax !== undefined) e.attrs.countMax = node.countMax; + this.applyMeta(e, node); + return e; + } + default: { + this.errors.push(`Unknown combinator '${String(node.op)}'`); + return seq(); + } + } + } + + /** Warn on an inverted `[min, max]` pair (a lower bound above its upper + * bound), which yields an unsatisfiable constraint downstream. */ + private checkBounds( + min: number | undefined, + max: number | undefined, + what: string, + minLabel: string, + maxLabel: string, + ): void { + if (min !== undefined && max !== undefined && min > max) { + this.warnings.push(`Inverted ${what} bounds: ${minLabel} (${min}) > ${maxLabel} (${max})`); + } + } + + /** + * Disambiguate duplicate explicit names among the direct children of a + * sequence/set. The solver turns each named child into a struct field keyed + * by its name; two siblings with the same name would silently collapse (the + * second overwrites the first, dropping a parameter). argtype is hand-authored + * and gives no uniqueness guarantee, so - like the mrtrix frontend - we rename + * collisions (`name_2`, `name_3`, ...) and warn. (This catches duplicate + * explicit labels; it does not detect collisions between solver-derived names + * of otherwise-unnamed children, which the solver also does not guard.) + */ + private dedupeSiblingNames(children: Expr[]): void { + const used = new Set(); + for (const child of children) { + const nm = child.meta?.name; + if (nm === undefined) continue; + if (!used.has(nm)) { + used.add(nm); + continue; + } + let n = 2; + while (used.has(`${nm}_${n}`)) n++; + const renamed = `${nm}_${n}`; + used.add(renamed); + child.meta = { ...child.meta, name: renamed }; + this.warnings.push(`Duplicate sibling name '${nm}'; renamed one occurrence to '${renamed}'`); + } + } + + /** `opt`/`rep` with multiple children implicitly wrap them in a sequence, + * which - like any sequence - is the output scope for those children. */ + private wrapChildren( + children: AstNode[], + expanding: Set, + name: string | undefined, + sink: Output[], + ): Expr { + if (children.length === 1) return this.lowerNode(children[0]!, expanding, name, sink); + const selfOutputs: Output[] = []; + const e = seq(...children.map((c) => this.lowerNode(c, expanding, name, selfOutputs))); + if (selfOutputs.length > 0) e.meta = { ...e.meta, outputs: selfOutputs }; + return e; + } + + /** Fold an AST node's name/doc/default decorations onto an IR node's meta. */ + private applyMeta(e: Expr, node: AstNode): void { + const meta: NodeMeta = {}; + if (node.name) meta.name = node.name; + const doc = docFrom(node); + if (doc) meta.doc = doc; + if (node.default !== undefined) meta.defaultValue = node.default; + if (Object.keys(meta).length > 0) e.meta = { ...e.meta, ...meta }; + } + + private toOutput(o: AstOutput, selfName: string | undefined): Output { + const tokens: OutputToken[] = []; + for (const t of o.tokens) { + if (t.kind === "literal") { + tokens.push({ kind: "literal", value: t.value }); + continue; + } + const targetName = t.name ?? selfName; + if (!targetName) { + this.warnings.push( + "Output self-reference '{}' has no named node to resolve to; emitting empty", + ); + tokens.push({ kind: "literal", value: "" }); + continue; + } + if (t.stripPrefix?.length) + this.warnings.push("Output ref op 'strip_prefix' is not supported by the IR; ignored"); + if (t.basename) + this.warnings.push("Output ref op 'basename' is not supported by the IR; ignored"); + tokens.push({ + kind: "ref", + target: nodeRef(targetName), + ...(t.stripSuffix?.length && { stripExtensions: t.stripSuffix }), + ...(t.or !== undefined && { fallback: t.or }), + }); + } + if (o.fallback !== undefined) { + this.warnings.push("Output-level '.or(...)' fallback is not supported by the IR; ignored"); + } + const doc = docFrom(o); + return { ...(o.name && { name: o.name }), ...(doc && { doc }), tokens }; + } +} + +export function buildAppMeta( + fm: Record | undefined, + rootDoc: { title?: string; description?: string }, + rootName: string | undefined, +): { meta?: AppMeta; warnings: string[] } { + const warnings: string[] = []; + const rootTitleDesc = docFrom(rootDoc); + if (!fm) { + // No frontmatter: id is the root definition name; still surface the root doc. + const meta: AppMeta = { + ...(rootName && { id: rootName }), + ...(rootTitleDesc && { doc: rootTitleDesc }), + }; + return Object.keys(meta).length > 0 ? { meta, warnings } : { warnings }; + } + + const asStr = (x: unknown): string | undefined => + typeof x === "string" && x.length > 0 ? x : undefined; + // The tool id is the root definition name; `exe` is the executable (argv[0]), + // which can differ (e.g. a `wb_command ` tool has id ``, exe + // `wb_command`). Fall back to `exe`/`id` frontmatter only if the root is unnamed. + const id = asStr(rootName) ?? asStr(fm.exe) ?? asStr(fm.id); + // An unquoted numeric version (`version: 6.0`) is parsed as a number by the + // frontmatter reader; accept it by stringifying rather than dropping it. + const version = + asStr(fm.version) ?? (typeof fm.version === "number" ? String(fm.version) : undefined); + const strList = (x: unknown): string[] => + Array.isArray(x) ? x.filter((a): a is string => typeof a === "string") : []; + const authors = strList(fm.authors); + const urls = strList(fm.urls); + const references = strList(fm.references); + + const doc: Documentation = { + ...rootTitleDesc, + ...(authors.length > 0 && { authors }), + ...(urls.length > 0 && { urls }), + ...(references.length > 0 && { literature: references }), + }; + + const meta: AppMeta = { + ...(id && { id }), + ...(version && { version }), + ...(Object.keys(doc).length > 0 && { doc }), + }; + + // container: { image, type } + if (isRecord(fm.container)) { + const image = asStr(fm.container.image); + const type = asStr(fm.container.type); + if (image) { + meta.container = { + image, + ...(type === "docker" || type === "singularity" ? { type } : {}), + }; + } + } + + meta.stdout = streamFrom(fm.stdout, warnings, "stdout"); + meta.stderr = streamFrom(fm.stderr, warnings, "stderr"); + if (!meta.stdout) delete meta.stdout; + if (!meta.stderr) delete meta.stderr; + + return { meta, warnings }; +} + +function streamFrom(x: unknown, warnings: string[], key: string): StreamOutput | undefined { + if (x === undefined || x === null) return undefined; + if (isRecord(x)) { + const name = typeof x.name === "string" ? x.name : undefined; + if (!name) { + warnings.push(`Frontmatter '${key}' is missing a 'name'; ignored`); + return undefined; + } + const description = typeof x.description === "string" ? x.description : undefined; + return { name, ...(description && { doc: { description } }) }; + } + if (typeof x === "string") return { name: x }; + warnings.push(`Frontmatter '${key}' has an unexpected shape; ignored`); + return undefined; +} + +function isRecord(x: unknown): x is Record { + return typeof x === "object" && x !== null && !Array.isArray(x); +} + +export function lowerDocument(doc: AstDocument): LowerResult { + const lowerer = new Lowerer(doc.aliases); + const result = lowerer.lower(doc); + + // argtype describes the arguments only (everything after argv[0]); the + // executable lives in frontmatter. Prepend it as the command's first token so + // the IR is a complete command line (the backend strips it back out). + const exe = typeof doc.frontmatter?.exe === "string" ? doc.frontmatter.exe : undefined; + if (exe) result.expr.attrs.nodes.unshift(lit(exe)); + + const { meta, warnings } = buildAppMeta( + doc.frontmatter, + { title: doc.root.title, description: doc.root.description }, + doc.rootName, + ); + return { + ...(meta && { meta }), + expr: result.expr, + errors: result.errors, + warnings: [...result.warnings, ...warnings], + }; +} diff --git a/packages/core/src/frontend/argtype/parser-frontend.ts b/packages/core/src/frontend/argtype/parser-frontend.ts new file mode 100644 index 0000000..8bdad6e --- /dev/null +++ b/packages/core/src/frontend/argtype/parser-frontend.ts @@ -0,0 +1,51 @@ +import type { Sequence } from "../../ir/node.js"; +import type { Frontend, ParseError, ParseResult, ParseWarning } from "../frontend.js"; +import { parseArgtype } from "./parser.js"; +import { lowerDocument } from "./lower.js"; + +/** A fresh empty root sequence for error returns (IR passes mutate in place). */ +function emptyExpr(): Sequence { + return { kind: "sequence", attrs: { nodes: [] } }; +} + +/** + * Frontend for the argtype sugar DSL - the hand-authored, TypeScript-types-like + * language for describing CLI argument grammars (see the argtype spec). Parses + * the text into an AST, then lowers it to Styx IR + AppMeta. + * + * Supported today: the argtype core (combinators, terminals, literals, naming, + * aliases, value constraints, `.join`/`.count`/`.default`, doc comments, + * frontmatter) plus the `outputs`, `mediatypes`, and `paths` extensions. `set` + * lowers to a sequence and `any` to its first branch; the `constraints` + * extension is parsed-and-ignored. + */ +export class ArgtypeParser implements Frontend { + readonly name = "argtype"; + readonly extensions = ["argtype"]; + + parse(source: string, _filename?: string): ParseResult { + const { doc, errors: parseErrors, warnings: parseWarnings } = parseArgtype(source); + const toLocation = (e: { line?: number; column?: number }) => + e.line !== undefined ? { location: { line: e.line, column: e.column } } : {}; + const errors: ParseError[] = parseErrors.map((e) => ({ message: e.message, ...toLocation(e) })); + const warnings: ParseWarning[] = parseWarnings.map((e) => ({ + message: e.message, + ...toLocation(e), + })); + + if (!doc) { + return { expr: emptyExpr(), errors, warnings }; + } + + const lowered = lowerDocument(doc); + warnings.push(...lowered.warnings.map((message) => ({ message }))); + errors.push(...lowered.errors.map((message) => ({ message }))); + + return { + ...(lowered.meta && { meta: lowered.meta }), + expr: lowered.expr, + errors, + warnings, + }; + } +} diff --git a/packages/core/src/frontend/argtype/parser.test.ts b/packages/core/src/frontend/argtype/parser.test.ts new file mode 100644 index 0000000..a00b9e0 --- /dev/null +++ b/packages/core/src/frontend/argtype/parser.test.ts @@ -0,0 +1,368 @@ +import { describe, expect, it } from "vitest"; +import { format } from "../../ir/format.js"; +import { ArgtypeParser } from "./parser-frontend.js"; +import { parseArgtype } from "./parser.js"; + +const parser = new ArgtypeParser(); + +function parse(src: string) { + const result = parser.parse(src); + return result; +} + +describe("argtype lexer/parser core", () => { + it("parses a bare positional terminal", () => { + const { errors, expr } = parse("echo: rep(message: str)"); + expect(errors).toEqual([]); + expect(expr.kind).toBe("sequence"); // root is wrapped? no - rep -> wrapped seq + }); + + it("lowers terminals, flags and value constraints", () => { + const r = parse(`convert: seq( + input: path, + set( + opt("-q", quality: int.min(1).max(100).default(80)), + opt("-v"), + ), + output: path, + )`); + expect(r.errors).toEqual([]); + const dump = format(r.expr, r.meta); + expect(dump).toContain("path [input]"); + expect(dump).toContain("int [quality] (1..100)"); + // set lowers to a sequence + expect(dump).toContain("sequence"); + }); + + it("resolves the alt infix operator with correct precedence", () => { + const r = parse(`tool: seq("--mode", mode: "fast" | "slow")`); + expect(r.errors).toEqual([]); + const dump = format(r.expr); + expect(dump).toContain("alternative [mode]"); + expect(dump).toContain('literal "fast"'); + expect(dump).toContain('literal "slow"'); + }); + + it("inlines type aliases", () => { + const r = parse(`Dimension = rep(int).count(2) +tool: seq(opt("-resize", size: Dimension))`); + expect(r.errors).toEqual([]); + const dump = format(r.expr); + expect(dump).toContain("repeat [size] {min=2, max=2}"); + }); + + it("detects recursive aliases", () => { + const r = parse(`A = seq(B) +B = seq(A) +tool: seq(A)`); + expect(r.errors.some((e) => /[Rr]ecursive alias/.test(e.message))).toBe(true); + }); + + it("treats `.join()` on an opt the same as on its inner seq (micro-syntax)", () => { + // `opt("K=", v).join()` collapses the flag+value into one `K=value` token, + // equivalent to putting the join on the inner sequence. + const onOpt = parse(`tool: seq(opt("K=", v: str).join())`); + const onSeq = parse(`tool: seq(opt(seq("K=", v: str).join()))`); + expect(onOpt.errors).toEqual([]); + expect(onOpt.warnings).toEqual([]); + expect(JSON.stringify(onOpt.expr)).toBe(JSON.stringify(onSeq.expr)); + // The join lands on the inner sequence, not lost. + expect(format(onOpt.expr)).toContain('sequence join=""'); + }); + + it("applies join and count modifiers", () => { + const r = parse(`cut: seq("-f", fields: rep(int).join(","))`); + expect(r.errors).toEqual([]); + const dump = format(r.expr); + expect(dump).toContain('repeat [fields] {join=","}'); + }); + + it("attaches doc comments to the following node", () => { + const ast = parseArgtype(`tool: seq( + /// The input image. + input: path, + )`); + expect(ast.doc?.root.children?.[0]?.description).toBe("The input image."); + }); + + it("supports .title() and .description() methods (set title/description directly)", () => { + const r = parse(`tool: seq(input: path.title("Input").description("The image to read."))`); + expect(r.errors).toEqual([]); + const input = + r.expr.kind === "sequence" ? r.expr.attrs.nodes.find((n) => n.kind === "path") : undefined; + expect(input?.meta?.doc?.title).toBe("Input"); + expect(input?.meta?.doc?.description).toBe("The image to read."); + + // `.description()` sets the text verbatim - a leading `#` is NOT a title here. + const r2 = parse(`tool: seq(x: str.description("# not a title"))`); + const x = + r2.expr.kind === "sequence" ? r2.expr.attrs.nodes.find((n) => n.kind === "str") : undefined; + expect(x?.meta?.doc?.title).toBeUndefined(); + expect(x?.meta?.doc?.description).toBe("# not a title"); + }); + + it("applies chained methods on an alias reference", () => { + const r = parse(`Coords = rep(float) +tool: seq(opt("-c", center: Coords.count(3)))`); + expect(r.errors).toEqual([]); + const dump = format(r.expr); + expect(dump).toContain("repeat [center] {min=3, max=3}"); + }); + + it("attaches a doc comment to an output entry", () => { + const r = parse(`--- +exe: "tool" +extensions: + - outputs +--- +tool: seq( + out: str, + opt("-m").output( + /// # Brain mask + /// + /// The binary brain mask file. + mask: \`{out}_mask.nii\`, + ), +)`); + expect(r.errors).toEqual([]); + const rootOutputs = r.expr.meta?.outputs ?? []; + const mask = rootOutputs.find((o) => o.name === "mask"); + expect(mask?.doc?.title).toBe("Brain mask"); + expect(mask?.doc?.description).toBe("The binary brain mask file."); + }); + + it("splits a doc block into title + description on the `# ` heading", () => { + const r = parse(`tool: seq( + /// # Short title + /// + /// The longer description + /// spanning two lines. + input: path, + /// A description with + /// + /// two paragraphs but no title. + output: path, + )`); + expect(r.errors).toEqual([]); + const nodes = r.expr.kind === "sequence" ? r.expr.attrs.nodes : []; + const input = nodes.find((n) => n.meta?.name === "input"); + const output = nodes.find((n) => n.meta?.name === "output"); + expect(input?.meta?.doc?.title).toBe("Short title"); + expect(input?.meta?.doc?.description).toBe("The longer description\nspanning two lines."); + // No leading `# ` -> all description, even with a blank line (no title). + expect(output?.meta?.doc?.title).toBeUndefined(); + expect(output?.meta?.doc?.description).toBe( + "A description with\n\ntwo paragraphs but no title.", + ); + }); + + it("parses quoted labels for non-identifier names", () => { + const r = parse(`"1deval": seq("1D": path, normal: str)`); + expect(r.errors).toEqual([]); + expect(r.expr.meta?.name).toBe("1deval"); + const nodes = r.expr.kind === "sequence" ? r.expr.attrs.nodes : []; + expect(nodes.find((n) => n.meta?.name === "1D")).toBeDefined(); + expect(nodes.find((n) => n.meta?.name === "normal")).toBeDefined(); + }); + + it("parses countMin/countMax including one-sided bounds", () => { + const r = parse(`tool: seq(a: rep(int).countMin(1).countMax(4), b: rep(str).countMax(2))`); + expect(r.errors).toEqual([]); + const nodes = r.expr.kind === "sequence" ? r.expr.attrs.nodes : []; + const a = nodes.find((n) => n.meta?.name === "a"); + const b = nodes.find((n) => n.meta?.name === "b"); + expect(a?.kind === "repeat" && a.attrs.countMin).toBe(1); + expect(a?.kind === "repeat" && a.attrs.countMax).toBe(4); + expect(b?.kind === "repeat" && b.attrs.countMin).toBeUndefined(); + expect(b?.kind === "repeat" && b.attrs.countMax).toBe(2); + }); + + it("parses scientific-notation numeric bounds", () => { + const r = parse(`tool: seq(x: float.min(2.2e-308).max(1e5))`); + expect(r.errors).toEqual([]); + const nodes = r.expr.kind === "sequence" ? r.expr.attrs.nodes : []; + const x = nodes.find((n) => n.meta?.name === "x"); + expect(x?.kind === "float" && x.attrs.minValue).toBe(2.2e-308); + expect(x?.kind === "float" && x.attrs.maxValue).toBe(1e5); + }); + + it("parses the paths extension methods onto path attrs", () => { + const r = parse(`tool: seq(image: path.mutable(), out: path.resolveParent())`); + expect(r.errors).toEqual([]); + expect(r.warnings).toEqual([]); + const nodes = r.expr.kind === "sequence" ? r.expr.attrs.nodes : []; + const image = nodes.find((n) => n.meta?.name === "image"); + const out = nodes.find((n) => n.meta?.name === "out"); + expect(image?.kind === "path" && image.attrs.mutable).toBe(true); + expect(out?.kind === "path" && out.attrs.resolveParent).toBe(true); + }); + + it("renames duplicate sibling names and warns", () => { + const r = parse(`tool: seq(output: path, output: path)`); + expect(r.errors).toEqual([]); + expect(r.warnings.some((w) => /Duplicate sibling name 'output'/.test(w.message))).toBe(true); + const dump = format(r.expr); + expect(dump).toContain("path [output]"); + expect(dump).toContain("path [output_2]"); + }); + + it("parses-and-ignores unsupported extension methods (constraints)", () => { + const r = parse(`tool: seq( + a: opt("-a"), + b: opt("-b").requires("a"), + )`); + // No hard error: an unimplemented extension must be ignorable. + expect(r.errors).toEqual([]); + expect( + r.warnings.some((w) => /Ignoring unsupported method '\.requires\(\)'/.test(w.message)), + ).toBe(true); + }); + + it("rejects `= default` after a method chain", () => { + const r = parse(`tool: seq(x: int.min(1) = 80)`); + expect(r.errors.some((e) => /only allowed on a bare terminal/.test(e.message))).toBe(true); + }); + + it("warns when `.join()` is used on an unsupported node", () => { + const r = parse(`tool: seq(x: path.join(","))`); + expect(r.warnings.some((w) => /only supported on seq\/set\/rep/.test(w.message))).toBe(true); + }); + + it("lowers `any` to its first branch (lossless for codegen, no warning)", () => { + const r = parse(`tool: seq(any("--output", "-output", "-o"), path)`); + expect(r.errors).toEqual([]); + expect(r.warnings).toEqual([]); + const dump = format(r.expr); + expect(dump).toContain('literal "--output"'); + expect(dump).not.toContain('literal "-output"'); + }); +}); + +describe("argtype frontmatter", () => { + it("parses frontmatter into AppMeta", () => { + const r = parse(`--- +exe: "bet" +version: "6.0.4" +authors: + - "FMRIB Analysis Group" +urls: + - "https://fsl.fmrib.ox.ac.uk" +--- + +/// Brain extraction. +bet: seq(infile: path)`); + expect(r.errors).toEqual([]); + expect(r.meta?.id).toBe("bet"); + expect(r.meta?.version).toBe("6.0.4"); + expect(r.meta?.doc?.authors).toEqual(["FMRIB Analysis Group"]); + expect(r.meta?.doc?.urls).toEqual(["https://fsl.fmrib.ox.ac.uk"]); + expect(r.meta?.doc?.description).toBe("Brain extraction."); + }); + + it("keeps body line numbers aligned after frontmatter", () => { + const r = parse(`--- +exe: "x" +--- +x: seq(@bad)`); + // The error should point at line 4 (the body), not line 1. + const loc = r.errors[0]?.location; + expect(loc?.line).toBe(4); + }); +}); + +describe("argtype parser: review regressions", () => { + it("has no `.doc()` method; it is an ignorable unknown method, not a description", () => { + const r = parse(`tool: seq(x: str.doc("The image."))`); + expect(r.errors).toEqual([]); + const x = + r.expr.kind === "sequence" ? r.expr.attrs.nodes.find((n) => n.kind === "str") : undefined; + expect(x?.meta?.doc?.description).toBeUndefined(); + expect(r.warnings.some((w) => /\.doc\(\)/.test(w.message))).toBe(true); + }); + + it("keeps an unquoted `#` inside a frontmatter value (URL fragment)", () => { + const r = parse(`--- +urls: + - https://fsl.fmrib.ox.ac.uk/wiki#bet +--- +bet: path`); + expect(r.errors).toEqual([]); + expect(r.meta?.doc?.urls).toEqual(["https://fsl.fmrib.ox.ac.uk/wiki#bet"]); + }); + + it("accepts an unquoted numeric version rather than dropping it", () => { + const r = parse(`--- +version: 6.0 +--- +bet: path`); + expect(r.errors).toEqual([]); + expect(r.meta?.version).toBe("6"); + }); + + it("flags a malformed number whose exponent has no digits", () => { + const r = parse(`tool: seq(x: int.min(1e))`); + expect(r.errors.some((e) => /exponent has no digits/.test(e.message))).toBe(true); + }); + + it("parses a quoted output ref name (non-identifier target)", () => { + const r = parse(`--- +extensions: + - outputs +--- +tool: seq("4d_output": path).output(result: \`{"4d_output"}.nii\`)`); + expect(r.errors).toEqual([]); + const ref = r.expr.meta?.outputs?.[0]?.tokens.find((t) => t.kind === "ref"); + expect(ref?.kind === "ref" ? ref.target.name : undefined).toBe("4d_output"); + }); + + it("warns on inverted value and repetition-count bounds", () => { + const v = parse(`tool: seq(x: int.min(5).max(2))`); + expect(v.warnings.some((w) => /Inverted value bounds/.test(w.message))).toBe(true); + const c = parse(`tool: rep(str).countMin(5).countMax(2)`); + expect(c.warnings.some((w) => /Inverted repetition count bounds/.test(w.message))).toBe(true); + }); + + it("ignores an unsupported output-template method with a warning, not an error", () => { + const r = parse(`--- +extensions: + - outputs +--- +tool: seq(x: path).output(result: \`{x}.nii\`.mystery("z"))`); + expect(r.errors).toEqual([]); + expect(r.warnings.some((w) => /mystery/.test(w.message))).toBe(true); + }); + + it("scans interpolations quote-aware and brace-balanced", () => { + // A `}` inside a quoted ref name, and nested `{}` inside a quoted op arg, + // must not prematurely end the interpolation. + const r = parse(`--- +extensions: + - outputs +--- +tool: seq("a}b": path, x: path).output( + one: \`{"a}b"}.nii\`, + two: \`{x.or("{fallback}")}.nii\`, +)`); + expect(r.errors).toEqual([]); + const outs = r.expr.meta?.outputs ?? []; + const refName = (i: number) => { + const t = outs[i]?.tokens.find((tk) => tk.kind === "ref"); + return t?.kind === "ref" ? t.target.name : undefined; + }; + expect(refName(0)).toBe("a}b"); + const two = outs[1]?.tokens.find((t) => t.kind === "ref"); + expect(two?.kind === "ref" ? two.fallback : undefined).toBe("{fallback}"); + }); + + it("accepts .title()/.description() on an output template", () => { + const r = parse(`--- +extensions: + - outputs +--- +tool: seq(x: path).output(mask: \`{x}.nii\`.title("Brain mask").description("The mask file."))`); + expect(r.errors).toEqual([]); + const out = r.expr.meta?.outputs?.[0]; + expect(out?.doc?.title).toBe("Brain mask"); + expect(out?.doc?.description).toBe("The mask file."); + }); +}); diff --git a/packages/core/src/frontend/argtype/parser.ts b/packages/core/src/frontend/argtype/parser.ts new file mode 100644 index 0000000..a3415b3 --- /dev/null +++ b/packages/core/src/frontend/argtype/parser.ts @@ -0,0 +1,455 @@ +/** + * Recursive-descent parser for the argtype sugar DSL, producing an `AstDocument` + * (frontmatter + aliases + a single root definition). Lowering to Styx IR is a + * separate step (`lower.ts`). + * + * Precedence, loosest to tightest: `,` (list separator) < `label:` < `|` (alt) + * < `.method()` (chaining). See the spec's "Sugar DSL" section. + */ + +import type { AstAlias, AstDocument, AstNode, AstOutput, Combinator, Terminal } from "./ast.js"; +import { lex } from "./lexer.js"; +import type { Token, TokenKind } from "./lexer.js"; +import { splitFrontmatter } from "./frontmatter.js"; +import { parseTemplate } from "./template.js"; +import { splitDocText } from "./doc.js"; + +/** Split a `///` block and attach its title/description to a node, unless the + * node already carries them (a leading doc and a chained `.title()`/ + * `.description()` don't both win - the first assignment stays). */ +function attachDocText(target: { title?: string; description?: string }, raw: string): void { + const { title, description } = splitDocText(raw); + if (title !== undefined && target.title === undefined) target.title = title; + if (description !== undefined && target.description === undefined) + target.description = description; +} + +const COMBINATORS = new Set(["seq", "set", "opt", "rep", "alt", "any"]); +const TERMINALS = new Set(["int", "float", "str", "path"]); + +export interface AstParseError { + message: string; + line?: number; + column?: number; +} + +export interface AstParseResult { + doc?: AstDocument; + errors: AstParseError[]; + warnings: AstParseError[]; +} + +/** Core + supported-extension chaining methods. Anything else is an extension + * we don't implement and is parsed-and-ignored (the spec's "ignorable" rule). */ +const KNOWN_METHODS = new Set([ + "name", + "title", + "description", + "default", + "min", + "max", + "join", + "count", + "countMin", + "countMax", + "mediaType", + "mutable", + "resolveParent", +]); + +class Parser { + private tokens: Token[]; + private pos = 0; + readonly errors: AstParseError[] = []; + readonly warnings: AstParseError[] = []; + + constructor(tokens: Token[]) { + this.tokens = tokens; + } + + private peek(o = 0): Token { + return this.tokens[Math.min(this.pos + o, this.tokens.length - 1)]!; + } + + private at(kind: TokenKind): boolean { + return this.peek().kind === kind; + } + + private next(): Token { + return this.tokens[this.pos++] ?? this.tokens[this.tokens.length - 1]!; + } + + private expect(kind: TokenKind, what: string): Token { + if (this.at(kind)) return this.next(); + const tok = this.peek(); + this.error(`Expected ${what} but found '${tok.value || tok.kind}'`, tok); + return tok; + } + + private error(message: string, tok: Token = this.peek()): void { + this.errors.push({ message, line: tok.line, column: tok.column }); + } + + private warn(message: string, tok: Token = this.peek()): void { + this.warnings.push({ message, line: tok.line, column: tok.column }); + } + + // -- Top level -- + + parseDocument(frontmatter: Record | undefined): AstDocument | undefined { + const aliases: AstAlias[] = []; + let rootName: string | undefined; + let root: AstNode | undefined; + + while (!this.at("eof")) { + const docs = this.collectDocs(); + + if (!this.at("ident") && !this.at("string")) { + this.error(`Expected a definition (name: expr) or alias (Name = expr)`); + break; + } + const nameTok = this.next(); + const quotedName = nameTok.kind === "string"; + + if (this.at("eq")) { + // Alias: Name = expr. Alias names must be identifiers (they are + // referenced by bare name at each use site). + if (quotedName) this.error("Alias names must be identifiers, not quoted strings", nameTok); + this.next(); + const expr = this.parseElement(); + if (docs) attachDocText(expr, docs); + aliases.push({ name: nameTok.value, expr }); + } else if (this.at("colon")) { + // Root definition: name: expr + this.next(); + const expr = this.parseElement(); + if (docs) attachDocText(expr, docs); + if (root) { + this.error( + `Multiple root definitions; '${nameTok.value}' ignored (already have '${rootName}')`, + nameTok, + ); + } else { + rootName = nameTok.value; + root = expr; + if (!root.name) root.name = nameTok.value; + } + } else { + this.error(`Expected ':' or '=' after '${nameTok.value}'`, nameTok); + break; + } + } + + if (!root || rootName === undefined) { + this.error("No root definition (expected `name: expr`)"); + return undefined; + } + return { ...(frontmatter && { frontmatter }), aliases, rootName, root }; + } + + /** Collect consecutive `///` doc lines, joined with newlines. */ + private collectDocs(): string | undefined { + const lines: string[] = []; + while (this.at("doc")) lines.push(this.next().value); + return lines.length > 0 ? lines.join("\n") : undefined; + } + + // -- Expressions -- + + /** + * An element is the unit inside a comma list: an optionally-named expression. + * `label:` is looser than `|`, so a name binds the whole alternative that + * follows it. + */ + private parseElement(): AstNode { + // A label is a bare identifier or a quoted string (for names that are not + // valid identifiers), followed by `:`. A quoted string NOT followed by `:` + // is a literal, so the colon lookahead disambiguates. + if ((this.at("ident") || this.at("string")) && this.peek(1).kind === "colon") { + const nameTok = this.next(); + this.next(); // colon + const inner = this.parseElement(); + inner.name = nameTok.value; + return inner; + } + return this.parseAlt(); + } + + private parseAlt(): AstNode { + const first = this.parseChain(); + if (!this.at("pipe")) return first; + const alts: AstNode[] = [first]; + while (this.at("pipe")) { + this.next(); + alts.push(this.parseChain()); + } + return { kind: "comb", op: "alt", children: alts }; + } + + private parseChain(): AstNode { + const node = this.parsePrimary(); + // Method chain. + let chained = false; + while (this.at("dot")) { + this.next(); + this.applyMethod(node); + chained = true; + } + // `= value` default sugar. The spec restricts it to a bare terminal: once a + // chain has started, `.default(...)` must be used instead. + if (this.at("eq")) { + const eqTok = this.next(); + const value = this.parseValue(); + if (node.kind === "terminal" && !chained) { + node.default = value; + } else { + this.error( + "`= value` default is only allowed on a bare terminal; use `.default(...)` after a method chain", + eqTok, + ); + } + } + return node; + } + + private parsePrimary(): AstNode { + const tok = this.peek(); + + if (tok.kind === "string") { + this.next(); + return { kind: "literal", value: tok.value }; + } + + if (tok.kind === "lparen") { + // Anonymous sequence group. + const children = this.parseParenList(); + return { kind: "comb", op: "seq", children }; + } + + if (tok.kind === "ident") { + if (COMBINATORS.has(tok.value as Combinator) && this.peek(1).kind === "lparen") { + this.next(); + const children = this.parseParenList(); + return { kind: "comb", op: tok.value as Combinator, children }; + } + if (TERMINALS.has(tok.value as Terminal)) { + this.next(); + return { kind: "terminal", terminal: tok.value as Terminal }; + } + // Otherwise an alias reference. + this.next(); + return { kind: "ref", refName: tok.value }; + } + + this.error(`Unexpected token '${tok.value || tok.kind}' in expression`, tok); + this.next(); + return { kind: "literal", value: "" }; + } + + /** Parse `( elem, elem, ... )` with optional leading docs and trailing comma. */ + private parseParenList(): AstNode[] { + this.expect("lparen", "'('"); + const items: AstNode[] = []; + while (!this.at("rparen") && !this.at("eof")) { + const docs = this.collectDocs(); + if (this.at("rparen")) break; // trailing docs before close + const elem = this.parseElement(); + if (docs) attachDocText(elem, docs); + items.push(elem); + if (this.at("comma")) this.next(); + else break; + } + this.expect("rparen", "')'"); + return items; + } + + // -- Method calls -- + + private applyMethod(node: AstNode): void { + const nameTok = this.expect("ident", "a method name"); + const method = nameTok.value; + if (method === "output") { + node.outputs = [...(node.outputs ?? []), ...this.parseOutputArgs()]; + return; + } + if (!KNOWN_METHODS.has(method)) { + // An extension method we don't implement (e.g. the draft `constraints` + // extension's `.requires()`/`.conflicts()`). The spec requires unknown + // extension annotations to be ignorable, so skip the argument list + // (whatever its shape) and continue rather than failing the parse. + this.skipBalancedArgs(); + this.warn(`Ignoring unsupported method '.${method}()'`, nameTok); + return; + } + + const args = this.parseScalarArgs(); + switch (method) { + case "name": + if (typeof args[0] === "string") node.name = args[0]; + break; + case "title": + if (typeof args[0] === "string") node.title = args[0]; + break; + case "description": + // `.description()` sets the description directly; the `# ` heading + // convention is sugar for the `///` form only, so a leading `#` here is a + // literal part of the description, not a title. + if (typeof args[0] === "string") node.description = args[0]; + break; + case "default": + if (args[0] !== undefined) node.default = args[0]; + break; + case "min": + if (typeof args[0] === "number") node.min = args[0]; + break; + case "max": + if (typeof args[0] === "number") node.max = args[0]; + break; + case "join": + node.join = typeof args[0] === "string" ? args[0] : ""; + break; + case "count": + // `.count(n)` is exactly n (sugar for `.countMin(n).countMax(n)`). For a + // one-sided or asymmetric bound, use `.countMin` / `.countMax` directly. + if (typeof args[0] === "number") { + node.countMin = args[0]; + node.countMax = args[0]; + } + break; + case "countMin": + if (typeof args[0] === "number") node.countMin = args[0]; + break; + case "countMax": + if (typeof args[0] === "number") node.countMax = args[0]; + break; + case "mediaType": + if (typeof args[0] === "string") (node.mediaTypes ??= []).push(args[0]); + break; + case "mutable": + node.mutable = true; + break; + case "resolveParent": + node.resolveParent = true; + break; + } + } + + /** Parse a parenthesized list of plain scalar arguments (numbers / strings). */ + private parseScalarArgs(): (string | number)[] { + this.expect("lparen", "'('"); + const args: (string | number)[] = []; + while (!this.at("rparen") && !this.at("eof")) { + args.push(this.parseValue()); + if (this.at("comma")) this.next(); + else break; + } + this.expect("rparen", "')'"); + return args; + } + + /** Consume a balanced `( ... )` group without interpreting its contents. + * Used to skip the arguments of an unsupported extension method. */ + private skipBalancedArgs(): void { + if (!this.at("lparen")) return; + let depth = 0; + do { + const tok = this.next(); + if (tok.kind === "lparen") depth++; + else if (tok.kind === "rparen") depth--; + else if (tok.kind === "eof") break; + } while (depth > 0); + } + + /** Parse the argument list of `.output(...)`: one or more template expressions. */ + private parseOutputArgs(): AstOutput[] { + this.expect("lparen", "'('"); + const outputs: AstOutput[] = []; + while (!this.at("rparen") && !this.at("eof")) { + const docs = this.collectDocs(); + if (this.at("rparen")) break; // trailing docs before close + const out = this.parseOutputTemplate(); + if (docs) attachDocText(out, docs); + outputs.push(out); + if (this.at("comma")) this.next(); + else break; + } + this.expect("rparen", "')'"); + return outputs; + } + + /** `label: \`tpl\`` or `\`tpl\`` followed by optional `.name(...)` / `.or(...)`. */ + private parseOutputTemplate(): AstOutput { + let name: string | undefined; + // The output name may be a bare identifier or a quoted label (for + // non-identifier output names), mirroring `label:` naming. + if ((this.at("ident") || this.at("string")) && this.peek(1).kind === "colon") { + name = this.next().value; + this.next(); // colon + } + + const out: AstOutput = { tokens: [] }; + if (this.at("template")) { + const tmplTok = this.next(); + const { tokens, errors } = parseTemplate(tmplTok.value); + out.tokens = tokens; + for (const e of errors) this.error(e, tmplTok); + } else { + this.error(`Expected an output template literal`); + } + + // Template-level chaining: .name("x") / .or("fallback") / .title("t") / + // .description("d"). An unrecognized method is an ignorable extension + // annotation (same rule as node method chains): skip its arguments and warn + // rather than failing the parse. + const CHAIN = new Set(["name", "or", "title", "description"]); + while (this.at("dot")) { + this.next(); + const m = this.expect("ident", "a template method"); + if (CHAIN.has(m.value)) { + const arg = this.parseScalarArgs()[0]; + if (typeof arg === "string") { + if (m.value === "name") name = arg; + else if (m.value === "or") out.fallback = arg; + else if (m.value === "title") out.title = arg; + else out.description = arg; // description + } + } else { + this.skipBalancedArgs(); + this.warn(`Ignoring unsupported output-template method '.${m.value}()'`, m); + } + } + + if (name !== undefined) out.name = name; + return out; + } + + private parseValue(): string | number { + const tok = this.peek(); + if (tok.kind === "string") { + this.next(); + return tok.value; + } + if (tok.kind === "number") { + this.next(); + return Number(tok.value); + } + this.error(`Expected a value (string or number) but found '${tok.value || tok.kind}'`, tok); + this.next(); + return ""; + } +} + +export function parseArgtype(source: string): AstParseResult { + const { frontmatter, body, errors: fmErrors } = splitFrontmatter(source); + const { tokens, errors: lexErrors } = lex(body); + + const parser = new Parser(tokens); + const doc = parser.parseDocument(frontmatter); + + const errors: AstParseError[] = [ + ...fmErrors.map((m) => ({ message: m })), + ...lexErrors.map((e) => ({ message: e.message, line: e.line, column: e.column })), + ...parser.errors, + ]; + return { ...(doc && { doc }), errors, warnings: parser.warnings }; +} diff --git a/packages/core/src/frontend/argtype/template.ts b/packages/core/src/frontend/argtype/template.ts new file mode 100644 index 0000000..33d5143 --- /dev/null +++ b/packages/core/src/frontend/argtype/template.ts @@ -0,0 +1,147 @@ +/** + * Parse an output template literal body (the text between backticks) into + * `AstOutputToken[]`. Literal runs become literal tokens; `{...}` interpolations + * become ref tokens carrying any reference operations (`strip_suffix`, + * `strip_prefix`, `basename`, `or`). + * + * `{}` (empty braces) is a self-reference: the value of the node the output is + * attached to. Lowering resolves it to that node's name. + */ + +import type { AstOutputToken } from "./ast.js"; + +export interface TemplateParseResult { + tokens: AstOutputToken[]; + errors: string[]; +} + +/** + * Index of the `}` that closes an interpolation opened at `open` (the position + * just past the `{`), or -1 if unterminated. A naive `indexOf("}")` would stop + * at the first `}` even when it sits inside a quoted ref name (`{"a}b"}`) or a + * quoted operation argument (`{a.or("{b}")}`); this scan skips double-quoted + * spans (honoring `\"`) and balances nested `{}` so those round-trip. + */ +function interpolationEnd(body: string, open: number): number { + let depth = 1; + let inQuote = false; + for (let j = open; j < body.length; j++) { + const c = body[j]!; + if (inQuote) { + if (c === "\\") + j++; // skip the escaped char (e.g. \" or \`) + else if (c === '"') inQuote = false; + continue; + } + if (c === '"') inQuote = true; + else if (c === "{") depth++; + else if (c === "}" && --depth === 0) return j; + } + return -1; +} + +export function parseTemplate(body: string): TemplateParseResult { + const tokens: AstOutputToken[] = []; + const errors: string[] = []; + let lit = ""; + let i = 0; + + const flushLit = (): void => { + if (lit.length > 0) { + tokens.push({ kind: "literal", value: lit }); + lit = ""; + } + }; + + while (i < body.length) { + const ch = body[i]!; + // A backslash escapes the next character: `\{`, `\}`, `` \` ``, `\\` become + // that literal char (so `{` need not start an interpolation). + if (ch === "\\" && i + 1 < body.length) { + const next = body[i + 1]!; + lit += next === "{" || next === "}" || next === "`" || next === "\\" ? next : "\\" + next; + i += 2; + continue; + } + if (ch === "{") { + const close = interpolationEnd(body, i + 1); + if (close === -1) { + errors.push("Unterminated '{' in output template"); + lit += body.slice(i); + break; + } + flushLit(); + const inner = body.slice(i + 1, close).trim(); + const { token, error } = parseRef(inner); + if (error) errors.push(error); + tokens.push(token); + i = close + 1; + } else { + lit += ch; + i++; + } + } + flushLit(); + return { tokens, errors }; +} + +/** Parse the inside of `{...}`: an optional name followed by `.op(arg)` calls. + * The name is a bare identifier, a double-quoted string (for a non-identifier + * target name, e.g. `{"4d_output"}`), or absent for a self-reference (`{}`). */ +function parseRef(inner: string): { token: AstOutputToken; error?: string } { + const token: AstOutputToken & { kind: "ref" } = { kind: "ref" }; + + let rest = inner; + const quoted = /^"((?:[^"\\]|\\.)*)"/.exec(inner); + if (quoted) { + // A quoted target name carries an arbitrary (non-identifier) name verbatim, + // mirroring a quoted `label:` (a `}` inside the quotes is fine - the scanner + // is quote-aware). A literal backtick still cannot appear in a name: the + // lexer ends the template at the first unescaped backtick. + token.name = unescapeArg(quoted[1]!); + rest = inner.slice(quoted[0].length); + } else { + // Leading identifier (the referenced name). + const nameMatch = /^[A-Za-z_][A-Za-z0-9_]*/.exec(inner); + if (nameMatch) { + token.name = nameMatch[0]; + rest = inner.slice(nameMatch[0].length); + } + } + + // Method chain: .op("arg") or .op() + const opRe = /^\.\s*([A-Za-z_]+)\s*\(\s*(?:"((?:[^"\\]|\\.)*)")?\s*\)/; + let error: string | undefined; + while (rest.length > 0) { + const m = opRe.exec(rest); + if (!m) { + error = `Unrecognized output-reference operation near '${rest}'`; + break; + } + const op = m[1]!; + const arg = m[2]; + switch (op) { + case "strip_suffix": + if (arg !== undefined) (token.stripSuffix ??= []).push(unescapeArg(arg)); + break; + case "strip_prefix": + if (arg !== undefined) (token.stripPrefix ??= []).push(unescapeArg(arg)); + break; + case "basename": + token.basename = true; + break; + case "or": + if (arg !== undefined) token.or = unescapeArg(arg); + break; + default: + error = `Unknown output-reference operation '${op}'`; + } + rest = rest.slice(m[0].length).trimStart(); + } + + return { token, ...(error && { error }) }; +} + +function unescapeArg(s: string): string { + return s.replace(/\\(.)/g, "$1"); +} diff --git a/packages/core/src/frontend/detect-format.test.ts b/packages/core/src/frontend/detect-format.test.ts index d9bfb3f..cd61865 100644 --- a/packages/core/src/frontend/detect-format.test.ts +++ b/packages/core/src/frontend/detect-format.test.ts @@ -44,8 +44,36 @@ describe("detectFormat", () => { expect(detectFormat(source)).toBe("mrtrix"); }); - it("returns null for invalid JSON", () => { + it("detects argtype by a combinator call", () => { + expect(detectFormat("bet: seq(infile: path)")).toBe("argtype"); + expect(detectFormat("x: opt(str)")).toBe("argtype"); + }); + + it("detects argtype by a leading frontmatter fence", () => { + expect(detectFormat('---\nexe: "bet"\n---\nbet: path')).toBe("argtype"); + }); + + it("detects argtype by a frontmatter fence after leading blank lines", () => { + // The parser tolerates blank lines before the fence, so detection must too. + expect(detectFormat('\n\n---\nexe: "bet"\n---\nbet: path')).toBe("argtype"); + }); + + it("detects a combinator-free argtype definition (bare terminal root)", () => { + expect(detectFormat("bet: path")).toBe("argtype"); + expect(detectFormat("x: int")).toBe("argtype"); + expect(detectFormat('greeting: "hello"')).toBe("argtype"); + expect(detectFormat('"1deval": path')).toBe("argtype"); + }); + + it("returns null for non-JSON prose that is not argtype", () => { expect(detectFormat("not json")).toBeNull(); + expect(detectFormat("the cat sat (here)")).toBeNull(); + }); + + it("never misclassifies valid JSON as argtype even if a string contains seq(", () => { + // Valid JSON parses successfully and never reaches the argtype text check. + const source = JSON.stringify({ foo: "this mentions seq( in a value" }); + expect(detectFormat(source)).toBeNull(); }); it("returns null for non-object JSON", () => { diff --git a/packages/core/src/frontend/detect-format.ts b/packages/core/src/frontend/detect-format.ts index 2f054d5..b28f5aa 100644 --- a/packages/core/src/frontend/detect-format.ts +++ b/packages/core/src/frontend/detect-format.ts @@ -1,7 +1,26 @@ -export type FormatName = "boutiques" | "argdump" | "workbench" | "mrtrix"; +export type FormatName = "boutiques" | "argdump" | "workbench" | "mrtrix" | "argtype"; + +/** Matches an argtype combinator call, e.g. `seq(` - the cheapest reliable + * marker that a non-JSON source is the argtype DSL. */ +const ARGTYPE_COMBINATOR = /\b(?:seq|set|opt|rep|alt|any)\s*\(/; + +/** Matches a top-level argtype definition (`name: expr` / `"name": expr`) whose + * right-hand side opens with a terminal, combinator, literal, template, or + * group - so a combinator-free spec like `bet: path` is still recognized. */ +const ARGTYPE_DEFINITION = + /^[ \t]*(?:[A-Za-z_]\w*|"(?:[^"\\]|\\.)*")[ \t]*:[ \t]*(?:(?:int|float|str|path|seq|set|opt|rep|alt|any)\b|["`(])/m; + +/** The first non-blank line of a source, trimmed (empty string if none). */ +function firstNonBlankLine(source: string): string { + for (const line of source.split("\n")) { + const trimmed = line.trim(); + if (trimmed !== "") return trimmed; + } + return ""; +} /** - * Auto-detect the format of a JSON descriptor source string. + * Auto-detect the format of a descriptor source string. * Returns null if the format cannot be determined. */ export function detectFormat(source: string): FormatName | null { @@ -9,6 +28,16 @@ export function detectFormat(source: string): FormatName | null { try { parsed = JSON.parse(source); } catch { + // Not JSON: the only non-JSON format is the argtype DSL. Recognize it by a + // `---` frontmatter fence (allowing leading blank lines, as the parser + // does), an argtype combinator call, or a top-level `name: expr` definition. + if ( + firstNonBlankLine(source) === "---" || + ARGTYPE_COMBINATOR.test(source) || + ARGTYPE_DEFINITION.test(source) + ) { + return "argtype"; + } return null; } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 86f9182..1ec647d 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,4 +1,5 @@ import { ArgdumpParser } from "./frontend/argdump/index.js"; +import { ArgtypeParser } from "./frontend/argtype/index.js"; import { BoutiquesParser } from "./frontend/boutiques/index.js"; import { MrtrixParser } from "./frontend/mrtrix/index.js"; import { WorkbenchParser } from "./frontend/workbench/index.js"; @@ -22,7 +23,10 @@ export function compile( ? { filename: filenameOrOptions } : (filenameOrOptions ?? {}); - const format = options.format ?? detectFormat(source); + // An explicit `.argtype` filename selects the DSL frontend even though the + // source is not JSON (detection by content is a fallback). + const byExtension = options.filename?.endsWith(".argtype") ? "argtype" : undefined; + const format = options.format ?? byExtension ?? detectFormat(source); if (!format) { return { @@ -32,13 +36,23 @@ export function compile( }; } - const parser = - format === "argdump" - ? new ArgdumpParser() - : format === "workbench" - ? new WorkbenchParser() - : format === "mrtrix" - ? new MrtrixParser() - : new BoutiquesParser(); + const parser = ((): { parse: (s: string, f?: string) => ParseResult } => { + switch (format) { + case "argdump": + return new ArgdumpParser(); + case "argtype": + return new ArgtypeParser(); + case "workbench": + return new WorkbenchParser(); + case "mrtrix": + return new MrtrixParser(); + case "boutiques": + return new BoutiquesParser(); + default: { + const _exhaustive: never = format; + return new BoutiquesParser(); + } + } + })(); return parser.parse(source, options.filename); } diff --git a/playground/public/examples/bet.argtype b/playground/public/examples/bet.argtype new file mode 100644 index 0000000..6a384c0 --- /dev/null +++ b/playground/public/examples/bet.argtype @@ -0,0 +1,103 @@ +--- +exe: "bet" +version: "6.0.4" +authors: + - "FMRIB Analysis Group, University of Oxford" +urls: + - "https://fsl.fmrib.ox.ac.uk/fsl/fslwiki" +extensions: + - outputs +--- + +/// Automated brain extraction tool for FSL +bet: seq( + /// Input image (e.g. img.nii.gz). + infile: path, + + /// Output brain mask prefix (e.g. img_bet). + maskfile: str = "img_bet", + + set( + /// Fractional intensity threshold (0->1). Smaller values give larger + /// brain outline estimates. + opt("-f", fractional_intensity: float.min(0).max(1).default(0.5)), + + /// Vertical gradient in fractional intensity threshold (-1->1). + /// Positive values give larger brain outline at bottom, smaller at top. + opt("-g", vg_fractional_intensity: float.min(-1).max(1).default(0)), + + /// XYZ coordinates (voxels, not mm) of the center of gravity of the + /// initial mesh surface. + opt("-c", center_of_gravity: rep(float).count(3)), + + /// Head radius (mm, not voxels); initial surface sphere is set to half + /// of this. + opt("-r", head_radius: float), + + /// Generate brain surface outline overlaid onto original image. + overlay: opt("-o").output(overlay_file: `{maskfile}_overlay.nii.gz`), + + /// Generate binary brain mask. + binary_mask_flag: opt("-m").output(binary_mask: `{maskfile}_mask.nii.gz`), + + /// Generate rough skull image (not as clean as betsurf). + approx_skull: opt("-s").output( + approx_skull_img: `{maskfile}_skull.nii.gz`, + skull_mask: `{maskfile}_skull_mask.nii.gz`, + ), + + /// Don't generate segmented brain image output. + no_seg_output: opt("-n"), + + /// Generate brain surface as mesh in .vtk format. + vtk_mesh: opt("-e").output(output_vtk_mesh: `{maskfile}_mesh.vtk`), + + /// Apply thresholding to segmented brain image and mask. + thresholding: opt("-t"), + + /// More robust brain center estimation by iterating BET with a + /// changing center-of-gravity. + robust_iters: opt("-R"), + + /// Cleanup residual eye and optic nerve voxels (useful before + /// SIENA/SIENAX). + residual_optic_cleanup: opt("-S"), + + /// Reduce image bias and residual neck voxels (useful before + /// SIENA/SIENAX). + reduce_bias: opt("-B"), + + /// Pad end slices in both directions; useful when only a few slices + /// are present in the Z direction. + slice_padding: opt("-Z"), + + /// Determine brain mask from the first volume of a 4D dataset and + /// apply it to the whole set. Intended for FMRI; the -f threshold + /// is reduced to 0.3. + whole_set_mask: opt("-F"), + + /// Run bet2 + betsurf so skull and scalp surfaces are also produced + /// (requires registration to standard space). + additional_surfaces: opt("-A").output( + out_inskull_mask: `{maskfile}_inskull_mask.nii.gz`, + out_inskull_mesh: `{maskfile}_inskull_mesh.nii.gz`, + out_inskull_off: `{maskfile}_inskull_mesh.off`, + out_outskin_mask: `{maskfile}_outskin_mask.nii.gz`, + out_outskin_mesh: `{maskfile}_outskin_mesh.nii.gz`, + out_outskin_off: `{maskfile}_outskin_mesh.off`, + out_outskull_mask: `{maskfile}_outskull_mask.nii.gz`, + out_outskull_mesh: `{maskfile}_outskull_mesh.nii.gz`, + out_outskull_off: `{maskfile}_outskull_mesh.off`, + ), + + /// Same as -A, but also takes a T2 image as input to refine the + /// skull/scalp surfaces. + opt("-A2", additional_surfaces_t2: path), + + /// Switch on diagnostic messages. + verbose: opt("-v"), + + /// Don't delete temporary intermediate images. + debug: opt("-d"), + ), +).output(outfile: `{maskfile}.nii.gz`) diff --git a/playground/src/App.svelte b/playground/src/App.svelte index 55f1123..1e66aa5 100644 --- a/playground/src/App.svelte +++ b/playground/src/App.svelte @@ -22,8 +22,14 @@ try { const res = await fetch(url); if (!res.ok) throw new Error(`HTTP ${res.status}`); - const jsonData = await res.json(); - input = JSON.stringify(jsonData, null, 2); + const text = await res.text(); + // JSON descriptors are pretty-printed; non-JSON sources (argtype) are + // shown verbatim. + try { + input = JSON.stringify(JSON.parse(text), null, 2); + } catch { + input = text; + } } catch (e) { input = `// Failed to load: ${e instanceof Error ? e.message : e}`; } finally { diff --git a/playground/src/lib/examples.ts b/playground/src/lib/examples.ts index b8aae54..3993a1d 100644 --- a/playground/src/lib/examples.ts +++ b/playground/src/lib/examples.ts @@ -43,6 +43,15 @@ export const exampleGroups: [ExampleGroup, ...ExampleGroup[]] = [ }, ], }, + { + label: "argtype", + examples: [ + { + name: "FSL bet", + url: import.meta.env.BASE_URL + "examples/bet.argtype", + }, + ], + }, ]; export const defaultExample = exampleGroups[0].examples[0]; diff --git a/playground/src/lib/tabs.ts b/playground/src/lib/tabs.ts index 4dc6fc5..17aac19 100644 --- a/playground/src/lib/tabs.ts +++ b/playground/src/lib/tabs.ts @@ -1,6 +1,7 @@ import { format, formatSolveResult, + ArgtypeBackend, BoutiquesBackend, JsonSchemaBackend, NipypeBackend, @@ -19,6 +20,7 @@ export interface TabDef { compute: (c: Compilation) => Map; } +const argtypeBackend = new ArgtypeBackend(); const boutiquesBackend = new BoutiquesBackend(); const jsonSchemaBackend = new JsonSchemaBackend(); const nipypeBackend = new NipypeBackend(); @@ -111,4 +113,12 @@ export const tabs: [TabDef, ...TabDef[]] = [ lang: "json", compute: (c) => emitWithPackage(boutiquesBackend, c), }, + { + id: "argtype", + label: "argtype", + // No dedicated shiki grammar; argtype is TypeScript-types-like, so borrow + // its highlighting (strings, comments, backtick templates render sensibly). + lang: "typescript" as BundledLanguage, + compute: (c) => emitWithPackage(argtypeBackend, c), + }, ];