sync-docs.ts

/**
 * Pulls docs/ from the sendra-lab/Sendra repo and rewrites it into this
 * repo's Docusaurus docs structure.
 *
 * Ref resolution
 * --------------
 * By default this syncs against sendra-lab/Sendra's live `main` branch HEAD,
 * resolved via the GitHub API at sync time — there's no pinned commit SHA to
 * bump. The redeploy is triggered automatically by a docs-sync workflow on
 * every push to sendra's main that touches docs/, so a stale pin would just
 * mean the deploy hook fires but re-syncs old content; tracking `main` live
 * is what makes the new content actually show up.
 *
 * This does mean an unrelated-looking sendra `main` push can change or break
 * a sendra-web build if it touches docs/ or the sync's assumptions about
 * doc structure — there's no reviewed pin standing between the two repos
 * anymore. The GitHub API call itself pins to a specific commit SHA *within*
 * a single sync run (the tree listing and every file fetch below all use the
 * same resolved `ref`), so a run is at least internally consistent even
 * though the ref changes from run to run.
 *
 * Override: set SENDRA_DOCS_REF to sync against a different ref (a branch,
 * tag, or commit) for local testing without editing this file.
 */

import { mkdir, rm, writeFile } from "node:fs/promises";
import path from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";

const SOURCE_OWNER = "sendra-lab";
const SOURCE_REPO = "Sendra";
const SOURCE_BRANCH = "main";
const SOURCE_DOCS_ROOT = "docs";

const scriptDir = path.dirname(fileURLToPath(import.meta.url));
const repoRoot = path.resolve(scriptDir, "..");
const DOCS_DIR = path.join(repoRoot, "docs");
// Synced content lives entirely under docs/cli/ so it never collides with
// hand-written docs elsewhere in docs/ (e.g. docs/intro.md). The whole
// directory is wiped and rewritten on every run, which is what makes the
// sync idempotent — nothing outside it is ever touched.
const SYNC_TARGET_DIR = path.join(DOCS_DIR, "cli");

const token = process.env.SENDRA_DOCS_TOKEN;

function githubHeaders(): Record<string, string> {
  const headers: Record<string, string> = {
    Accept: "application/vnd.github+json",
    "User-Agent": "sendra-web-docs-sync",
  };
  if (token) headers.Authorization = `Bearer ${token}`;
  return headers;
}

/**
 * Resolves SOURCE_BRANCH to its current commit SHA. Every file fetched in
 * this run (tree listing, raw content, edit-URL/link generation) uses this
 * one resolved SHA, not the floating branch name, so a single sync run is
 * never split across two different commits even if `main` moves mid-run.
 */
async function resolveHeadSha(): Promise<string> {
  const url = `https://api.github.com/repos/${SOURCE_OWNER}/${SOURCE_REPO}/commits/${SOURCE_BRANCH}`;
  const res = await fetch(url, {
    headers: { ...githubHeaders(), Accept: "application/vnd.github.sha" },
  });
  if (!res.ok) {
    throw new Error(
      `Failed to resolve ${SOURCE_OWNER}/${SOURCE_REPO}@${SOURCE_BRANCH} HEAD: ${res.status} ${res.statusText}\n` +
        (await res.text()),
    );
  }
  return (await res.text()).trim();
}

// Set at the top of main() — every function below that reads `ref` is only
// ever called after that assignment runs.
let ref: string;

interface TreeEntry {
  path: string;
  type: "blob" | "tree";
  sha: string;
}

interface GitTreeResponse {
  tree: TreeEntry[];
  truncated: boolean;
}

async function fetchDocsFileList(): Promise<string[]> {
  const url = `https://api.github.com/repos/${SOURCE_OWNER}/${SOURCE_REPO}/git/trees/${ref}?recursive=1`;
  const res = await fetch(url, { headers: githubHeaders() });
  if (!res.ok) {
    throw new Error(
      `Failed to list ${SOURCE_OWNER}/${SOURCE_REPO}@${ref} tree: ${res.status} ${res.statusText}\n` +
        (await res.text()),
    );
  }
  const data = (await res.json()) as GitTreeResponse;
  if (data.truncated) {
    throw new Error(
      "GitHub tree API response was truncated — repo is too large for a single recursive listing.",
    );
  }
  return data.tree
    .filter((entry) => entry.type === "blob")
    .map((entry) => entry.path)
    .filter(
      (p) => p.startsWith(`${SOURCE_DOCS_ROOT}/`) && p.endsWith(".md"),
    );
}

/**
 * Date of the most recent commit (at `ref`) that touched sendra's docs/,
 * used as the "Last updated" date on every synced page. It's one API call
 * for the whole docs set, not one per file: a per-file date would need a
 * request per doc, and the unauthenticated GitHub API allows only 60 an
 * hour — enough to start failing the tree listing this sync depends on.
 * The trade-off is that a page shows when the docs last changed, not when
 * that page did.
 *
 * Best-effort: returns null (pages just show no date) rather than failing
 * the sync if GitHub can't answer.
 */
async function fetchDocsLastUpdated(): Promise<string | null> {
  const url = `https://api.github.com/repos/${SOURCE_OWNER}/${SOURCE_REPO}/commits?sha=${ref}&path=${SOURCE_DOCS_ROOT}&per_page=1`;
  try {
    const res = await fetch(url, { headers: githubHeaders() });
    if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
    const commits = (await res.json()) as Array<{
      commit: { committer: { date: string } };
    }>;
    return commits[0]?.commit.committer.date ?? null;
  } catch (error) {
    console.warn(
      `Could not look up when ${SOURCE_DOCS_ROOT}/ last changed; synced pages will show no "Last updated" date: ${error}`,
    );
    return null;
  }
}

async function fetchRawFile(sourcePath: string): Promise<string> {
  const url = `https://raw.githubusercontent.com/${SOURCE_OWNER}/${SOURCE_REPO}/${ref}/${sourcePath}`;
  const res = await fetch(url, token ? { headers: { Authorization: `Bearer ${token}` } } : undefined);
  if (!res.ok) {
    throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
  }
  return res.text();
}

/** Path relative to docs/, e.g. "reference/requests.md" or "reference.md". */
type DocsRelPath = string;

function toDocsRelPath(fullPath: string): DocsRelPath {
  return fullPath.slice(`${SOURCE_DOCS_ROOT}/`.length);
}

/**
 * Maps a source docs/-relative path to this repo's output path (relative to
 * docs/cli/). `README.md` becomes `index.md` per Docusaurus's convention for
 * a folder's landing page. `reference.md` is special-cased the same way even
 * though it isn't literally named README.md: it's sendra's own topic index
 * for the sibling reference/ folder (a bullet list linking into it), so it
 * belongs *inside* reference/ as that category's landing page — left as a
 * sibling file, it and the reference/ folder both show up in the sidebar
 * labeled "Reference", which is confusing rather than merely sparse.
 */
function toOutputRelPath(sourceRel: DocsRelPath): string {
  if (sourceRel === "reference.md") return "reference/index.md";
  const base = path.posix.basename(sourceRel, ".md");
  const dir = path.posix.dirname(sourceRel); // "." for top-level files
  const outBase = base === "README" ? "index" : base;
  return dir === "." ? `${outBase}.md` : path.posix.join(dir, `${outBase}.md`);
}

/** Docusaurus route (site-root-relative) for a docs/-relative source path. */
function toDocRoute(sourceRel: DocsRelPath): string {
  const outRel = toOutputRelPath(sourceRel).replace(/\.md$/, "");
  const withoutIndex = outRel.replace(/(^|\/)index$/, "");
  const routeSuffix = withoutIndex === "" ? "" : `/${withoutIndex}`;
  return `/docs/cli${routeSuffix}`;
}

function extractTitle(body: string): string | undefined {
  const match = body.match(/^#\s+(.+)$/m);
  return match?.[1]?.trim();
}

/** Order derived from reference.md / decisions/README.md's own bullet lists. */
function extractOrder(indexBody: string, indexDir: string): Map<string, number> {
  const order = new Map<string, number>();
  const linkPattern = /\]\(([^)#]+\.md)(#[^)]*)?\)/g;
  let i = 0;
  let match: RegExpExecArray | null;
  while ((match = linkPattern.exec(indexBody))) {
    const target = path.posix.normalize(path.posix.join(indexDir, match[1]));
    if (!order.has(target)) order.set(target, ++i);
  }
  return order;
}

function resolveLinkTarget(sourceRel: DocsRelPath, linkPath: string): string {
  const sourceDir = path.posix.dirname(sourceRel);
  return path.posix.normalize(path.posix.join(sourceDir, linkPath));
}

function rewriteLinks(
  body: string,
  sourceRel: DocsRelPath,
  knownDocs: Set<string>,
): string {
  // Matches every markdown link target: [text](target). Docusaurus's broken-
  // link check validates *any* relative link, not just ones ending in .md —
  // sendra's docs also link out to source files (../../sendra-core/src/...)
  // and example fixtures (../../examples/auth.yaml) that this sync doesn't
  // pull in, so every relative link needs handling, not just doc-to-doc ones.
  return body.replace(
    /(\]\()([^)\s]+)(\))/g,
    (full, open: string, target: string, close: string) => {
      // Leave scheme URLs (http:, https:, mailto:, ...) untouched.
      if (/^[a-z][a-z0-9+.-]*:/i.test(target)) return full;
      // Leave same-page anchors and already-absolute paths untouched.
      if (target.startsWith("#") || target.startsWith("/")) return full;

      const hashIdx = target.indexOf("#");
      const linkPath = hashIdx === -1 ? target : target.slice(0, hashIdx);
      const fragment = hashIdx === -1 ? "" : target.slice(hashIdx);
      if (linkPath === "") return full;

      const resolved = resolveLinkTarget(sourceRel, linkPath);

      if (linkPath.endsWith(".md") && knownDocs.has(resolved)) {
        const route = toDocRoute(resolved);
        return `${open}${route}${fragment}${close}`;
      }

      // Anything else — a doc outside docs/ (../README.md), or a non-doc
      // file (source, example fixture) — isn't synced, so link straight at
      // the pinned commit on GitHub instead of a Docusaurus route that
      // would 404.
      const repoPath = path.posix.normalize(
        path.posix.join(SOURCE_DOCS_ROOT, path.posix.dirname(sourceRel), linkPath),
      );
      const githubUrl = `https://github.com/${SOURCE_OWNER}/${SOURCE_REPO}/blob/${ref}/${repoPath}${fragment}`;
      return `${open}${githubUrl}${close}`;
    },
  );
}

function buildFrontMatter(opts: {
  id: string;
  title: string;
  sidebarPosition?: number;
  editUrl: string;
  lastUpdated: string | null;
}): string {
  const lines = [
    "---",
    `id: ${opts.id}`,
    `title: "${opts.title.replace(/"/g, '\\"')}"`,
    `custom_edit_url: ${opts.editUrl}`,
  ];
  // These files are generated and untracked here, so git has no history to
  // date them from; this front matter is what the "Last updated" line reads.
  if (opts.lastUpdated) {
    lines.push("last_update:", `  date: "${opts.lastUpdated}"`);
  }
  lines.push(
    // sendra's docs use `{{variable}}` templating and `<https://...>`
    // autolinks — valid CommonMark, not valid JSX. `mdx.format: md` opts
    // just this synced file out of MDX's JSX-aware parser; the site default
    // (full MDX, for hand-written docs and blog posts) is untouched. Must be
    // nested under `mdx:` — Docusaurus reads `frontMatter.mdx.format`, not a
    // top-level `format` key (see @docusaurus/mdx-loader's utils.js).
    "mdx:",
    "  format: md",
  );
  if (opts.sidebarPosition !== undefined) {
    lines.push(`sidebar_position: ${opts.sidebarPosition}`);
  }
  lines.push("---", "");
  return lines.join("\n");
}

interface CategoryMeta {
  label: string;
  position: number;
  collapsed: boolean;
  link:
    | { type: "doc"; id: string }
    | { type: "generated-index"; description?: string };
}

async function writeCategory(dir: string, meta: CategoryMeta): Promise<void> {
  await mkdir(dir, { recursive: true });
  await writeFile(
    path.join(dir, "_category_.json"),
    `${JSON.stringify(meta, null, 2)}\n`,
    "utf8",
  );
}

async function main() {
  ref = process.env.SENDRA_DOCS_REF ?? (await resolveHeadSha());
  console.log(`Syncing docs/ from ${SOURCE_OWNER}/${SOURCE_REPO}@${ref}...`);

  const filePaths = await fetchDocsFileList();
  if (filePaths.length === 0) {
    throw new Error("No markdown files found under docs/ at the pinned ref — refusing to sync.");
  }

  const sourceRelPaths = filePaths.map(toDocsRelPath);
  const knownDocs = new Set(sourceRelPaths);

  const contents = new Map<DocsRelPath, string>();
  for (const rel of sourceRelPaths) {
    contents.set(rel, await fetchRawFile(`${SOURCE_DOCS_ROOT}/${rel}`));
  }

  const lastUpdated = await fetchDocsLastUpdated();

  const topOrder = extractOrder(contents.get("reference.md") ?? "", ".");
  const decisionsOrder = extractOrder(contents.get("decisions/README.md") ?? "", "decisions");

  // Clean previous run's output. Scoped to SYNC_TARGET_DIR only, so any
  // hand-written docs living elsewhere under docs/ are never touched.
  await rm(SYNC_TARGET_DIR, { recursive: true, force: true });
  await mkdir(SYNC_TARGET_DIR, { recursive: true });

  for (const rel of sourceRelPaths) {
    const raw = contents.get(rel)!;
    const title = extractTitle(raw) ?? path.basename(rel, ".md");
    const rewritten = rewriteLinks(raw, rel, knownDocs);

    const outRel = toOutputRelPath(rel);
    // Docusaurus ids may not contain a slash — location within docs/cli/
    // already disambiguates identically-named files in different folders
    // (e.g. reference/exit-codes.md vs decisions/exit-codes.md), so the
    // bare basename is all `id` needs to be.
    const id = path.posix.basename(outRel, ".md");
    // The two folder-index docs (reference/index.md, decisions/index.md) get
    // their place in the sidebar from their _category_.json's own
    // `position` below, not from a per-doc sidebar_position.
    const sidebarPosition = rel.startsWith("reference/")
      ? topOrder.get(rel)
      : rel.startsWith("decisions/")
        ? decisionsOrder.get(rel)
        : undefined;

    // "Edit this page" should send contributors to the actual source (the
    // sendra repo's main branch), not to sendra-web — editing the synced
    // copy here would be silently discarded on the next `pnpm sync-docs`.
    const editUrl = `https://github.com/${SOURCE_OWNER}/${SOURCE_REPO}/edit/main/${SOURCE_DOCS_ROOT}/${rel}`;

    const frontMatter = buildFrontMatter({
      id,
      title,
      sidebarPosition,
      editUrl,
      lastUpdated,
    });
    const banner =
      `<!-- Auto-generated by \`pnpm sync-docs\` from ` +
      `${SOURCE_OWNER}/${SOURCE_REPO}@${ref.slice(0, 12)}:${SOURCE_DOCS_ROOT}/${rel}. ` +
      `Do not edit directly — edit the source in the sendra repo and re-run the sync. -->\n\n`;

    const outputPath = path.join(SYNC_TARGET_DIR, outRel);
    await mkdir(path.dirname(outputPath), { recursive: true });
    await writeFile(outputPath, frontMatter + banner + rewritten, "utf8");
  }

  // Category metadata for the autogenerated sidebar. Written fresh every
  // run (SYNC_TARGET_DIR is wiped above) since it lives in the same synced
  // tree as the docs it describes.
  // "Documentation", not "CLI Reference" — sendra ships both `run`/`test`
  // (CLI) and `sendra tui` (interactive TUI), and reference/tui.md lives
  // right alongside the CLI-flag docs, so a CLI-scoped label would misdescribe
  // half the category's own contents.
  await writeCategory(SYNC_TARGET_DIR, {
    label: "Documentation",
    position: 2, // after hand-written docs/intro.md (sidebar_position: 1)
    collapsed: false,
    link: {
      type: "generated-index",
      description:
        "Reference and design-decision docs for Sendra, synced from the sendra-lab/Sendra repo.",
    },
  });
  await writeCategory(path.join(SYNC_TARGET_DIR, "reference"), {
    label: "Reference",
    position: 1,
    collapsed: false,
    link: { type: "doc", id: "index" },
  });
  await writeCategory(path.join(SYNC_TARGET_DIR, "decisions"), {
    label: "Design Decisions",
    position: 2,
    collapsed: false,
    link: { type: "doc", id: "index" },
  });

  console.log(`Synced ${sourceRelPaths.length} file(s) into ${path.relative(repoRoot, SYNC_TARGET_DIR)}/`);
}

main().catch((err) => {
  console.error(err);
  process.exitCode = 1;
});