From 9ddb95f8e07215708a44f263f78029b67606f49d Mon Sep 17 00:00:00 2001 From: Jack Hsu Date: Thu, 6 Aug 2026 12:10:31 -0400 Subject: [PATCH 1/2] chore(nx-dev): sync docs kb articles into the pylon knowledge base ## Current Behavior Pylon crawls nx.dev, but crawled training data does not feed the support widget's suggested-answer feature. Only real KB articles are eligible. The 173 articles left in Pylon from the closed #36277 migration are stale copies of pre-DOC-552 doc paths, with no way to refresh them. ## Expected Behavior Nightly job mirrors `astro-docs/src/content/docs/kb` into Pylon. nx.dev stays canonical - copy only, no source deletion, no redirects. - `markdoc-to-html.ts` converts the Markdoc AST to Pylon-safe HTML. All 184 articles convert with no warnings. - `pylon-client.ts` wraps the REST API with retry, throttle and pagination. - `sync-kb.ts` diffs source against live and creates/updates. Pylon stores `body_html` verbatim, so change detection is a byte compare - no state file. - Images carry a `data-nx-src` content-hash marker so re-runs reuse the uploaded CDN URL instead of re-uploading. - Writes are confined to one collection, so the hand-written enterprise articles beside them cannot be touched. - `--prune` is opt-in and refuses to delete more than a quarter of the collection. Articles stay unlisted to keep nx.dev SEO-canonical. Needs `PYLON_API_TOKEN` in CI. ## Related Issue(s) Fixes DOC-542 --- .github/workflows/pylon-kb-sync.yml | 68 +++ astro-docs/project.json | 16 + astro-docs/scripts/pylon/markdoc-to-html.ts | 541 ++++++++++++++++++++ astro-docs/scripts/pylon/pylon-client.ts | 207 ++++++++ astro-docs/scripts/pylon/render-preview.ts | 87 ++++ astro-docs/scripts/pylon/sync-kb.ts | 335 ++++++++++++ 6 files changed, 1254 insertions(+) create mode 100644 .github/workflows/pylon-kb-sync.yml create mode 100644 astro-docs/scripts/pylon/markdoc-to-html.ts create mode 100644 astro-docs/scripts/pylon/pylon-client.ts create mode 100644 astro-docs/scripts/pylon/render-preview.ts create mode 100644 astro-docs/scripts/pylon/sync-kb.ts diff --git a/.github/workflows/pylon-kb-sync.yml b/.github/workflows/pylon-kb-sync.yml new file mode 100644 index 00000000000..d0caed9fdac --- /dev/null +++ b/.github/workflows/pylon-kb-sync.yml @@ -0,0 +1,68 @@ +name: Pylon KB Sync + +# Mirrors astro-docs/src/content/docs/kb into the Pylon knowledge base so the +# support widget can suggest these articles as answers. nx.dev stays canonical. + +on: + schedule: + - cron: '0 7 * * *' + workflow_dispatch: + inputs: + dry-run: + description: 'Report changes without writing to Pylon' + type: boolean + default: false + prune: + description: 'Delete Pylon articles whose source page no longer exists' + type: boolean + default: false + +permissions: {} + +jobs: + sync: + if: ${{ github.repository_owner == 'nrwl' }} + permissions: + contents: read # to fetch code (actions/checkout) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Setup dev tools with mise + uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3 + + - name: Enable corepack and install pnpm + run: | + corepack enable + corepack prepare --activate + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Sync knowledge base + working-directory: astro-docs + env: + PYLON_API_TOKEN: ${{ secrets.PYLON_API_TOKEN }} + PYLON_AUTHOR_USER_ID: ${{ vars.PYLON_AUTHOR_USER_ID }} + run: | + pnpm exec tsx scripts/pylon/sync-kb.ts \ + ${{ inputs.dry-run && '--dry-run' || '' }} \ + ${{ inputs.prune && '--prune' || '' }} + + report: + if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }} + needs: sync + runs-on: ubuntu-latest + name: Report status + steps: + - name: Send notification + uses: ravsamhq/notify-slack-action@be814b201e233b2dc673608aa46e5447c8ab13f2 # v11 + with: + status: ${{ needs.sync.result }} + message_format: '{emoji} Pylon KB sync has {status_message}' + notification_title: '{workflow}' + footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>' + notify_when: 'failure' + env: + SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} diff --git a/astro-docs/project.json b/astro-docs/project.json index 4aae99d5d1d..7296713a436 100644 --- a/astro-docs/project.json +++ b/astro-docs/project.json @@ -171,6 +171,22 @@ "{projectRoot}/scripts/vale-changed.mjs" ] }, + "pylon-sync": { + "//": "Mirrors src/content/docs/kb into the Pylon knowledge base. Needs PYLON_API_TOKEN. Runs nightly via .github/workflows/pylon-kb-sync.yml", + "cache": false, + "command": "tsx scripts/pylon/sync-kb.ts", + "options": { + "cwd": "astro-docs" + } + }, + "pylon-preview": { + "//": "Renders kb articles through the Pylon converter offline, no API token needed", + "cache": false, + "command": "tsx scripts/pylon/render-preview.ts", + "options": { + "cwd": "astro-docs" + } + }, "format": { "cache": true, "//": "nx format doesn't respect overrides, so we manually run prettier for mdoc files", diff --git a/astro-docs/scripts/pylon/markdoc-to-html.ts b/astro-docs/scripts/pylon/markdoc-to-html.ts new file mode 100644 index 00000000000..2b6bb97beed --- /dev/null +++ b/astro-docs/scripts/pylon/markdoc-to-html.ts @@ -0,0 +1,541 @@ +/** + * Converts a `/docs/kb` Markdoc article into the HTML Pylon stores as an + * article body. + * + * Pylon persists `body_html` verbatim - create, patch and publish all round + * trip byte for byte - so the output is compared directly against the live + * article to decide whether a push is needed. Anything emitted here must + * therefore be deterministic. + * + * Interactive tags (project graphs, generated card indexes) have no HTML + * equivalent and become a pointer back to the canonical nx.dev page. + */ + +import Markdoc, { type Node } from '@markdoc/markdoc'; +import { createHash } from 'node:crypto'; +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, relative, resolve } from 'node:path'; + +/** An image the sync has to make reachable from Pylon's CDN. */ +export interface AssetRef { + /** + * Identity of the image contents, embedded in the emitted `data-nx-src` so a + * later run can reuse the URL already uploaded for it. + */ + key: string; + absolutePath: string; + fileName: string; + contentType: string; +} + +export interface ConvertOptions { + /** Absolute path of the `.mdoc` file, used to resolve relative image paths. */ + sourcePath: string; + /** Absolute path of `astro-docs/public`, the root for `/`-prefixed images. */ + publicDir: string; + /** Repository root, used to build stable asset keys. */ + repoRoot: string; + /** Canonical page on nx.dev, e.g. `https://nx.dev/docs/kb/caching`. */ + canonicalUrl: string; + /** Origin used to absolutize root-relative links, e.g. `https://nx.dev`. */ + siteUrl: string; +} + +export interface ConvertResult { + title: string; + description: string; + /** Image `src` values are `nx-asset:` placeholders the caller resolves. */ + html: string; + assets: AssetRef[]; + warnings: string[]; +} + +export const ASSET_PLACEHOLDER_PREFIX = 'nx-asset:'; + +const CONTENT_TYPES: Record = { + avif: 'image/avif', + gif: 'image/gif', + jpeg: 'image/jpeg', + jpg: 'image/jpeg', + png: 'image/png', + svg: 'image/svg+xml', + webp: 'image/webp', +}; + +/** Fallback headings for `aside`/`callout` blocks that carry no title. */ +const CALLOUT_LABELS: Record = { + announcement: 'Announcement', + caution: 'Caution', + check: 'Check', + danger: 'Danger', + deepdive: 'Deep dive', + note: 'Note', + tip: 'Tip', + warning: 'Warning', +}; + +export function convertArticle( + source: string, + options: ConvertOptions +): ConvertResult { + const ast = Markdoc.parse(stripHtmlComments(source)); + const frontmatter = parseFrontmatter(ast.attributes?.frontmatter ?? ''); + const converter = new Converter(options); + const body = converter.renderChildren(ast); + + return { + title: frontmatter.title, + description: frontmatter.description, + html: body + converter.renderCanonicalFooter(), + assets: [...converter.assets.values()], + warnings: converter.warnings, + }; +} + +class Converter { + readonly assets = new Map(); + readonly warnings: string[] = []; + #options: ConvertOptions; + + constructor(options: ConvertOptions) { + this.#options = options; + } + + renderCanonicalFooter(): string { + return ( + `

This article mirrors the Nx documentation. ` + + `Read the latest version at ` + + `${escapeText(this.#options.canonicalUrl)}.

` + ); + } + + renderChildren(node: Node): string { + return (node.children ?? []).map((child) => this.render(child)).join(''); + } + + render(node: Node): string { + const attributes = (node.attributes ?? {}) as Record; + + switch (node.type) { + case 'document': + case 'inline': + return this.renderChildren(node); + + case 'text': + return escapeText(String(attributes.content ?? '')); + + case 'softbreak': + return '\n'; + case 'hardbreak': + return '
'; + + case 'paragraph': + return `

${this.renderChildren(node)}

`; + + case 'heading': { + // Pylon renders the article title as the page h1, so demote any h1. + const level = Math.min(Math.max(Number(attributes.level ?? 2), 2), 6); + return `${this.renderChildren(node)}`; + } + + case 'strong': + return `${this.renderChildren(node)}`; + case 'em': + return `${this.renderChildren(node)}`; + case 's': + return `${this.renderChildren(node)}`; + + case 'code': + return `${escapeText(String(attributes.content ?? ''))}`; + + case 'fence': + return this.#renderFence(attributes); + + case 'link': { + const href = this.#absolutizeLink(String(attributes.href ?? '')); + return `${this.renderChildren(node)}`; + } + + case 'image': + return this.#renderImage(attributes); + + case 'list': + return attributes.ordered + ? `
    ${this.renderChildren(node)}
` + : `
    ${this.renderChildren(node)}
`; + case 'item': + return `
  • ${unwrapSingleParagraph(this.renderChildren(node))}
    • `; + + case 'blockquote': + return `
    ${this.renderChildren(node)}
    `; + + case 'table': + return `${this.renderChildren(node)}
    `; + case 'thead': + return `${this.renderChildren(node)}`; + case 'tbody': + return `${this.renderChildren(node)}`; + case 'tr': + return `${this.renderChildren(node)}`; + case 'th': + case 'td': { + const align = attributes.align + ? ` align="${escapeAttribute(String(attributes.align))}"` + : ''; + return `<${node.type}${align}>${this.renderChildren(node)}`; + } + + case 'hr': + return '
    '; + + case 'comment': + case 'error': + return ''; + + case 'tag': + return this.#renderTag(node, attributes); + + default: + this.warnings.push(`unhandled node type "${node.type}"`); + return this.renderChildren(node); + } + } + + #renderFence(attributes: Record): string { + const language = String(attributes.language ?? '').trim(); + const languageClass = language + ? ` class="language-${escapeAttribute(language)}"` + : ''; + const code = `
    ${escapeText(String(attributes.content ?? ''))}
    `; + + // `title` names the file or terminal the snippet belongs to; keep it as a + // caption. `meta` carries line-highlight ranges with no HTML equivalent. + const title = attributes.title ? String(attributes.title) : ''; + return title ? `

    ${escapeText(title)}

    ${code}` : code; + } + + #renderImage(attributes: Record): string { + const src = String(attributes.src ?? ''); + const alt = escapeAttribute(String(attributes.alt ?? '')); + + const absolutePath = src.startsWith('/') + ? resolve(this.#options.publicDir, src.slice(1)) + : resolve(dirname(this.#options.sourcePath), src); + + if (!existsSync(absolutePath)) { + this.warnings.push(`image not found on disk: ${src}`); + return `${alt}`; + } + + const extension = absolutePath.split('.').pop()?.toLowerCase() ?? ''; + const repoRelative = relative(this.#options.repoRoot, absolutePath); + const digest = createHash('sha256') + .update(readFileSync(absolutePath)) + .digest('hex') + .slice(0, 16); + const key = `${repoRelative}#${digest}`; + + this.assets.set(key, { + key, + absolutePath, + fileName: absolutePath.split('/').pop() ?? 'image', + contentType: CONTENT_TYPES[extension] ?? 'application/octet-stream', + }); + + return ( + `` + ); + } + + #renderTag(node: Node, attributes: Record): string { + const attribute = (name: string): string => + attributes[name] === undefined ? '' : String(attributes[name]); + + switch (node.tag) { + // Starlight asides and the Nx callout render the same way: a quoted + // block led by its title. + case 'aside': + case 'callout': { + const label = + attribute('title') || + CALLOUT_LABELS[attribute('type')] || + CALLOUT_LABELS.note; + return ( + `

    ${escapeText(label)}

    ` + + `${this.renderChildren(node)}
    ` + ); + } + + // Tab groups flatten to sequential sections - each panel keeps its label + // as a heading so the alternatives stay distinguishable. + case 'tabs': + return this.renderChildren(node); + case 'tabitem': + return ( + `

    ${escapeText(attribute('label'))}

    ` + + this.renderChildren(node) + ); + + // A file tree is already authored as a nested list. + case 'filetree': + return this.renderChildren(node); + + case 'youtube': { + const embedUrl = toYoutubeEmbedUrl(attribute('src')); + const caption = attribute('caption'); + return ( + `` + + (caption ? `

    ${escapeText(caption)}

    ` : '') + ); + } + + case 'course_video': + return this.#renderLinkParagraph( + attribute('courseUrl') || attribute('src'), + attribute('courseTitle') || 'Watch the course' + ); + + case 'github_repository': + return this.#renderLinkParagraph( + attribute('url'), + attribute('title') || attribute('url') + ); + + case 'call_to_action': { + const description = attribute('description'); + return ( + `

    ${this.#renderLink(attribute('url'), attribute('title'))}` + + (description ? ` - ${escapeText(description)}` : '') + + `

    ` + ); + } + + // Card grids are link lists once the layout is gone. + case 'cards': + case 'cardgrid': + return `
      ${this.renderChildren(node)}
    `; + case 'card': + case 'linkcard': { + const description = attribute('description'); + return ( + `
  • ${this.#renderLink(attribute('url') || attribute('href'), attribute('title'))}` + + (description ? `: ${escapeText(description)}` : '') + + `
    • ` + ); + } + + case 'badge': + return `${escapeText(attribute('text'))}`; + + // A prompt is meant to be copied verbatim, so keep it preformatted. + // `{pageUrl}` is substituted on nx.dev with the page's markdown source. + case 'llm_copy_prompt': { + const prompt = plainText(node) + .replaceAll('{pageUrl}', `${this.#options.canonicalUrl}.md`) + .trim(); + return ( + `

    ${escapeText(attribute('title'))}

    ` + + `
    ${escapeText(prompt)}
    ` + ); + } + + // Hidden from human readers on nx.dev; keep it that way here. + case 'llm_only': + return ''; + + // No static equivalent - point at the page that can render them. + case 'graph': + case 'project_details': + case 'index_page_cards': + return this.#renderCanonicalPointer(node.tag, attribute('title')); + + default: + this.warnings.push(`unsupported Markdoc tag "${node.tag}"`); + return this.renderChildren(node); + } + } + + #renderCanonicalPointer(tag: string, title: string): string { + const labels: Record = { + graph: 'project graph', + project_details: 'project details view', + index_page_cards: 'list of related pages', + }; + const label = title || labels[tag] || tag; + return ( + `

    This section shows an interactive ${escapeText(label)}. ` + + `View it on nx.dev.

    ` + ); + } + + #renderLinkParagraph(href: string, label: string): string { + return `

    ${this.#renderLink(href, label)}

    `; + } + + #renderLink(href: string, label: string): string { + const url = this.#absolutizeLink(href); + return `${escapeText(label || url)}`; + } + + /** + * Pylon serves these articles off help.nx.app, so every in-repo link has to + * become absolute against nx.dev. Bare fragments resolve against the + * canonical page rather than the Pylon article, whose heading anchors differ. + */ + #absolutizeLink(href: string): string { + if (!href) return this.#options.canonicalUrl; + if (/^[a-z][a-z0-9+.-]*:/i.test(href) || href.startsWith('//')) return href; + if (href.startsWith('#')) return this.#options.canonicalUrl + href; + if (href.startsWith('/')) return this.#options.siteUrl + href; + + this.warnings.push(`relative link left unresolved: ${href}`); + return href; + } +} + +/** + * Markdoc keeps `` as ordinary text, so without this the site's + * editorial notes and vale directives would show up as article copy. Comments + * inside fenced blocks are real sample code and must survive. + */ +function stripHtmlComments(source: string): string { + const lines: string[] = []; + let openFence: string | null = null; + let insideComment = false; + + for (const line of source.split('\n')) { + const fence = line.match(/^\s*(`{3,}|~{3,})/)?.[1]; + + if (openFence) { + lines.push(line); + if ( + fence && + fence[0] === openFence[0] && + fence.length >= openFence.length + ) { + openFence = null; + } + continue; + } + + if (fence && !insideComment) { + openFence = fence; + lines.push(line); + continue; + } + + let remaining = line; + let kept = ''; + while (remaining) { + if (insideComment) { + const end = remaining.indexOf('-->'); + if (end === -1) break; + insideComment = false; + remaining = remaining.slice(end + '-->'.length); + } else { + const start = remaining.indexOf('