feat: use auto-generated docs in the system prompt (#45)

evals/eval-with-docs.eval.ts

@@ -0,0 +1,18 @@
+import { getSystemPromptWithDocs } from "../lib/get-system-prompt-with-docs"
+
+/**
+ * Re-exports the system prompt with auto-generated docs for use in evals.
+ * Import `systemPrompt` from this module wherever you need the full prompt.
+ *
+ * Example usage in an eval file:
+ *
+ * ```ts
+ * import { systemPrompt } from "../evals/eval-with-docs.eval"
+ *
+ * const result = await aiModel.complete({
+ *   system: systemPrompt,
+ *   prompt: "Create a simple LED circuit with a current-limiting resistor",
+ * })
+ * ```
+ */
+export const systemPrompt = getSystemPromptWithDocs()

lib/get-system-prompt-with-docs.ts

@@ -0,0 +1,58 @@
+import * as fs from "fs"
+import * as path from "path"
+
+const DOCS_CACHE_PATH = path.join(
+  process.cwd(),
+  "assets",
+  "tscircuit-docs.md"
+)
+
+/**
+ * Reads the auto-generated tscircuit docs from the local cache file.
+ * The cache is populated by running `npx tsx scripts/update-docs.ts`.
+ */
+function readCachedDocs(): string {
+  if (fs.existsSync(DOCS_CACHE_PATH)) {
+    return fs.readFileSync(DOCS_CACHE_PATH, "utf-8")
+  }
+  return ""
+}
+
+/**
+ * Returns the base system prompt text (without docs) as a plain string.
+ * This is the static portion that instructs the model on tscircuit basics.
+ */
+export function getBaseSystemPrompt(): string {
+  return `You are an expert at generating tscircuit code. tscircuit is a TypeScript library for designing electronic circuits using a React-like JSX/TSX syntax.
+
+## Rules
+- Always import components from "tscircuit"
+- Use a \`<board>\` element as the root
+- Specify component footprints using the \`footprint\` prop
+- Connect pins using shared net name strings
+- All coordinates and dimensions are in millimeters
+- Export the circuit as a default function
+- Do not use placeholder values; use real electrical values and footprints`
+}
+
+/**
+ * Returns the full system prompt including auto-generated component docs.
+ * The docs section is appended after the base prompt.
+ */
+export function getSystemPromptWithDocs(): string {
+  const base = getBaseSystemPrompt()
+  const docs = readCachedDocs()
+
+  if (!docs) {
+    return base
+  }
+
+  return `${base}
+
+## tscircuit Component Reference (Auto-Generated Docs)
+
+The following documentation is auto-generated from the tscircuit source and provides
+accurate, up-to-date API references for all available components and their props.
+
+${docs}`
+}

lib/get-system-prompt.ts

@@ -0,0 +1,112 @@
+import * as fs from "fs"
+import * as path from "path"
+
+/**
+ * Fetches the auto-generated tscircuit docs from the registry or local cache.
+ * These docs are generated from the tscircuit source and provide up-to-date
+ * component/prop references for the AI model.
+ */
+async function fetchAutoGeneratedDocs(): Promise<string> {
+  // Try to load from local assets first (cached copy)
+  const localDocsPath = path.join(
+    process.cwd(),
+    "assets",
+    "tscircuit-docs.md"
+  )
+  if (fs.existsSync(localDocsPath)) {
+    return fs.readFileSync(localDocsPath, "utf-8")
+  }
+
+  // Fall back to fetching from the tscircuit docs endpoint
+  try {
+    const response = await fetch(
+      "https://registry-api.tscircuit.com/autorouting/docs/auto-generated"
+    )
+    if (response.ok) {
+      return await response.text()
+    }
+  } catch (e) {
+    // ignore fetch errors, fall through to empty string
+  }
+
+  return ""
+}
+
+/**
+ * Builds the system prompt used for AI-based circuit generation benchmarks.
+ * Incorporates auto-generated tscircuit component documentation so the model
+ * has accurate, up-to-date API references.
+ */
+export async function getSystemPrompt(): Promise<string> {
+  const autoGeneratedDocs = await fetchAutoGeneratedDocs()
+
+  const docsSection = autoGeneratedDocs
+    ? `\n\n## tscircuit Component Reference (Auto-Generated)\n\n${autoGeneratedDocs}`
+    : ""
+
+  return `You are an expert at generating tscircuit code. tscircuit is a TypeScript library for designing electronic circuits using React-like syntax.
+
+## Core Concepts
+
+- Use JSX/TSX syntax to describe circuits
+- Components represent electronic parts (resistors, capacitors, LEDs, etc.)
+- Connections are made using the \`net\` prop or by sharing net names
+- Layouts are specified using props like \`pcbX\`, \`pcbY\`, \`schX\`, \`schY\`
+
+## Key Rules
+
+1. Always import from "tscircuit": \`import { resistor, capacitor, led, ... } from "tscircuit"\`
+2. Use the \`<board>\` component as the root element
+3. Specify footprints using the \`footprint\` prop (e.g., \`footprint="0402"\`)
+4. Connect components using shared net names (strings) on their pin props
+5. Use \`<trace>\` components for explicit routing when needed
+6. All measurements are in millimeters unless otherwise specified
+
+## Example Circuit
+
+\`\`\`tsx
+import { resistor, led, battery } from "tscircuit"
+
+export default () => (
+  <board width="10mm" height="10mm">
+    <battery
+      name="B1"
+      footprint="cr2032"
+      pcbX={0}
+      pcbY={0}
+      positive="VCC"
+      negative="GND"
+    />
+    <resistor
+      name="R1"
+      resistance="330"
+      footprint="0402"
+      pcbX={3}
+      pcbY={0}
+      pin1="VCC"
+      pin2="led_anode"
+    />
+    <led
+      name="LED1"
+      footprint="0402"
+      color="red"
+      pcbX={6}
+      pcbY={0}
+      anode="led_anode"
+      cathode="GND"
+    />
+  </board>
+)
+\`\`\`
+${docsSection}
+
+## Instructions
+
+- Generate complete, valid tscircuit code that compiles without errors
+- Include all necessary imports
+- Use realistic component values and footprints
+- Place components at reasonable PCB positions to avoid overlaps
+- Always wrap everything in a default export function returning a \`<board>\` element
+- Do not use placeholder values — provide real electrical values
+`
+}

lib/system-prompt.ts

@@ -0,0 +1,64 @@
+import * as fs from "fs"
+import * as path from "path"
+
+/**
+ * Reads the auto-generated tscircuit documentation from the local assets folder.
+ * These docs are generated from the official tscircuit source and cover all
+ * available components, their props, and usage examples.
+ *
+ * To refresh the docs, run:
+ *   npx tsx scripts/update-docs.ts
+ */
+function getAutoGeneratedDocs(): string {
+  const docsPath = path.join(process.cwd(), "assets", "tscircuit-docs.md")
+  if (fs.existsSync(docsPath)) {
+    return fs.readFileSync(docsPath, "utf-8")
+  }
+  return ""
+}
+
+/**
+ * The main system prompt used in all tscircuit prompt benchmarks.
+ *
+ * Includes:
+ * 1. Core instructions for generating valid tscircuit code
+ * 2. Auto-generated component documentation for accurate API references
+ */
+export function getSystemPrompt(): string {
+  const autoGeneratedDocs = getAutoGeneratedDocs()
+
+  const docsBlock = autoGeneratedDocs
+    ? `\n\n<tscircuit_docs>\n${autoGeneratedDocs}\n</tscircuit_docs>`
+    : ""
+
+  return `You are an expert electronics engineer and tscircuit developer.
+tscircuit is an open-source TypeScript library that lets you design PCBs and schematics
+using React/JSX syntax.
+
+## Coding Rules
+
+1. Import all components from \`"tscircuit"\`
+2. The root element must always be \`<board>\`
+3. Specify footprints via the \`footprint\` prop (e.g. \`footprint="0402"\`, \`footprint="soic8"\`)
+4. Connect component pins by giving them the same net name string
+5. Use \`pcbX\` / \`pcbY\` (mm) to position components on the PCB
+6. Use \`schX\` / \`schY\` to position components on the schematic
+7. Export the circuit as a default function returning JSX
+8. Use real component values — no placeholders
+9. Avoid component overlaps by spacing parts at least 2 mm apart
+
+## Example
+
+\`\`\`tsx
+import { resistor, led, battery } from "tscircuit"
+
+export default () => (
+  <board width="20mm" height="10mm">
+    <battery name="B1" footprint="cr2032" pcbX={0} pcbY={0} positive="VCC" negative="GND" />
+    <resistor name="R1" resistance="330" footprint="0402" pcbX={5} pcbY={0} pin1="VCC" pin2="net1" />
+    <led name="D1" footprint="0402" color="red" pcbX={10} pcbY={0} anode="net1" cathode="GND" />
+  </board>
+)
+\`\`\`
+${docsBlock}`
+}

scripts/update-docs.ts

@@ -0,0 +1,63 @@
+/**
+ * Script to fetch and cache the auto-generated tscircuit documentation.
+ * Run this periodically to keep the docs up-to-date.
+ *
+ * Usage: npx tsx scripts/update-docs.ts
+ */
+
+import * as fs from "fs"
+import * as path from "path"
+
+const DOCS_OUTPUT_PATH = path.join(process.cwd(), "assets", "tscircuit-docs.md")
+
+const DOCS_SOURCES = [
+  // tscircuit registry auto-generated docs
+  "https://registry-api.tscircuit.com/autorouting/docs/auto-generated",
+  // fallback: GitHub raw content
+  "https://raw.githubusercontent.com/tscircuit/tscircuit/main/docs/AUTO_GENERATED.md",
+]
+
+async function fetchDocs(): Promise<string | null> {
+  for (const url of DOCS_SOURCES) {
+    try {
+      console.log(`Trying to fetch docs from: ${url}`)
+      const response = await fetch(url)
+      if (response.ok) {
+        const text = await response.text()
+        if (text.trim().length > 0) {
+          console.log(`✓ Successfully fetched docs from: ${url}`)
+          return text
+        }
+      } else {
+        console.warn(`✗ Failed to fetch from ${url}: HTTP ${response.status}`)
+      }
+    } catch (e) {
+      console.warn(`✗ Error fetching from ${url}:`, e)
+    }
+  }
+  return null
+}
+
+async function main() {
+  const docs = await fetchDocs()
+
+  if (!docs) {
+    console.error("Failed to fetch docs from any source.")
+    process.exit(1)
+  }
+
+  // Ensure the assets directory exists
+  const assetsDir = path.dirname(DOCS_OUTPUT_PATH)
+  if (!fs.existsSync(assetsDir)) {
+    fs.mkdirSync(assetsDir, { recursive: true })
+  }
+
+  fs.writeFileSync(DOCS_OUTPUT_PATH, docs, "utf-8")
+  console.log(`✓ Docs written to: ${DOCS_OUTPUT_PATH}`)
+  console.log(`  Size: ${(docs.length / 1024).toFixed(1)} KB`)
+}
+
+main().catch((e) => {
+  console.error(e)
+  process.exit(1)
+})