diff --git a/lib/prompt-templates/create-local-circuit-prompt.ts b/lib/prompt-templates/create-local-circuit-prompt.ts index a93f11f..a05dce2 100644 --- a/lib/prompt-templates/create-local-circuit-prompt.ts +++ b/lib/prompt-templates/create-local-circuit-prompt.ts @@ -3,6 +3,7 @@ import { getFootprintSizes, fp, } from "@tscircuit/footprinter" +import { fetchTSCircuitAIDocs } from "../utils/fetch-tscircuit-context" async function fetchFileContent(url: string): Promise { try { @@ -44,12 +45,17 @@ export const createLocalCircuitPrompt = async () => { .join("\n") .replace(/\n\n+/g, "\n\n") + // Fetch comprehensive TSCircuit AI documentation + const tscircuitAIDocs = await fetchTSCircuitAIDocs() + return ` You are an expert in electronic circuit design and tscircuit, and your job is to create a circuit board in tscircuit with the user-provided description. YOU MUST ABIDE BY THE RULES IN THE RULES SECTION -## tscircuit API overview +## TSCircuit Comprehensive Documentation + +${tscircuitAIDocs ? `${tscircuitAIDocs}\n\n` : ""}## tscircuit API overview Here's an overview of the tscircuit API: @@ -330,7 +336,7 @@ export const MyLed = () => ( // ### Importing Components // You can import a variety of components from the tscircuit registry. tscircuit -// registry components are always prefixed with \`@tsci/\`. Make sure to include +// registry components are always prefixed with `@tsci/`. Make sure to include // your imports at the top of the codefence. // If you are not told explicitly that an import exists, do not import it. diff --git a/lib/utils/fetch-tscircuit-context.ts b/lib/utils/fetch-tscircuit-context.ts new file mode 100644 index 0000000..0dc32f9 --- /dev/null +++ b/lib/utils/fetch-tscircuit-context.ts @@ -0,0 +1,31 @@ +/** + * Utility to fetch TSCircuit AI documentation + */ + +export async function fetchTSCircuitAIDocs(): Promise { + try { + console.log( + "Fetching TSCircuit AI documentation from docs.tscircuit.com/ai.txt", + ) + + const response = await fetch("https://docs.tscircuit.com/ai.txt", { + headers: { + "User-Agent": "tscircuit-prompt-benchmarks", + }, + }) + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`) + } + + const docs = await response.text() + console.log( + `Successfully fetched TSCircuit AI docs (${docs.length} characters)`, + ) + return docs + } catch (error) { + console.warn("Failed to fetch TSCircuit AI docs:", error) + console.log("Continuing without TSCircuit AI context") + return "" + } +} diff --git a/lib/utils/get-primary-source-code-from-vfs.ts b/lib/utils/get-primary-source-code-from-vfs.ts index e3af300..79715d0 100644 --- a/lib/utils/get-primary-source-code-from-vfs.ts +++ b/lib/utils/get-primary-source-code-from-vfs.ts @@ -1,12 +1,26 @@ export function getPrimarySourceCodeFromVfs( vfs: Record | undefined, ): string | undefined { - let code = undefined - if (vfs && Object.keys(vfs).length !== 0) { - const finalResultKey = Object.keys(vfs)[Object.keys(vfs).length - 1] - if (finalResultKey.includes("final")) { - code = vfs[finalResultKey] - } + if (!vfs || Object.keys(vfs).length === 0) { + return undefined } - return code + + const finalKeys = Object.keys(vfs).filter((key) => key.includes("final")) + + if (finalKeys.length === 0) { + return undefined + } + + const parsePromptId = (key: string): number => { + const match = key.match(/prompt-(\d+)-attempt-final/) + return match ? parseInt(match[1], 10) : 0 + } + + finalKeys.sort((a, b) => { + const idA = parsePromptId(a) + const idB = parsePromptId(b) + return idB - idA + }) + + return vfs[finalKeys[0]] } diff --git a/tests/tscircuitCoder.test.ts b/tests/tscircuitCoder.test.ts index d66c022..09875cf 100644 --- a/tests/tscircuitCoder.test.ts +++ b/tests/tscircuitCoder.test.ts @@ -2,38 +2,41 @@ import { createTscircuitCoder } from "lib/tscircuit-coder/tscircuitCoder" import { expect, test } from "bun:test" import { getPrimarySourceCodeFromVfs } from "lib/utils/get-primary-source-code-from-vfs" -test("TscircuitCoder submitPrompt streams and updates vfs", async () => { - const streamedChunks: string[] = [] - let vfsUpdated = false - const tscircuitCoder = createTscircuitCoder() - tscircuitCoder.on("streamedChunk", (chunk: string) => { - streamedChunks.push(chunk) - }) - tscircuitCoder.on("vfsChanged", () => { - vfsUpdated = true - }) +test.skipIf(!process.env.OPENAI_API_KEY)( + "TscircuitCoder submitPrompt streams and updates vfs", + async () => { + const streamedChunks: string[] = [] + let vfsUpdated = false + const tscircuitCoder = createTscircuitCoder() + tscircuitCoder.on("streamedChunk", (chunk: string) => { + streamedChunks.push(chunk) + }) + tscircuitCoder.on("vfsChanged", () => { + vfsUpdated = true + }) - await tscircuitCoder.submitPrompt({ - prompt: "create bridge rectifier circuit", - }) + await tscircuitCoder.submitPrompt({ + prompt: "create bridge rectifier circuit", + }) - await tscircuitCoder.submitPrompt({ - prompt: "add a transistor component", - }) + await tscircuitCoder.submitPrompt({ + prompt: "add a transistor component", + }) - let codeWithTransistor = getPrimarySourceCodeFromVfs(tscircuitCoder.vfs) - expect(codeWithTransistor).toInclude("transistor") + let codeWithTransistor = getPrimarySourceCodeFromVfs(tscircuitCoder.vfs) + expect(codeWithTransistor).toInclude("transistor") - await tscircuitCoder.submitPrompt({ - prompt: "add a tssop20 chip", - }) + await tscircuitCoder.submitPrompt({ + prompt: "add a tssop20 chip", + }) - let codeWithChip = getPrimarySourceCodeFromVfs(tscircuitCoder.vfs) - expect(codeWithChip).toInclude("tssop20") - expect(codeWithChip).toInclude("transistor") + let codeWithChip = getPrimarySourceCodeFromVfs(tscircuitCoder.vfs) + expect(codeWithChip).toInclude("tssop20") + expect(codeWithChip).toInclude("transistor") - expect(streamedChunks.length).toBeGreaterThan(0) - const vfsKeys = Object.keys(tscircuitCoder.vfs) - expect(vfsKeys.length).toBeGreaterThan(0) - expect(vfsUpdated).toBe(true) -}) + expect(streamedChunks.length).toBeGreaterThan(0) + const vfsKeys = Object.keys(tscircuitCoder.vfs) + expect(vfsKeys.length).toBeGreaterThan(0) + expect(vfsUpdated).toBe(true) + }, +) diff --git a/tests/utils/generate-random-prompts.test.ts b/tests/utils/generate-random-prompts.test.ts index 41a061c..4670f2f 100644 --- a/tests/utils/generate-random-prompts.test.ts +++ b/tests/utils/generate-random-prompts.test.ts @@ -2,10 +2,13 @@ import { describe, it, expect } from "bun:test" import { generateRandomPrompts } from "../../lib/utils/generate-random-prompts" describe("generateRandomPrompts", () => { - it("should return an array of prompts", async () => { - const prompts = await generateRandomPrompts(3) + it.skipIf(!process.env.OPENAI_API_KEY)( + "should return an array of prompts", + async () => { + const prompts = await generateRandomPrompts(3) - expect(Array.isArray(prompts)).toBe(true) - expect(prompts.length).toBe(3) - }) + expect(Array.isArray(prompts)).toBe(true) + expect(prompts.length).toBe(3) + }, + ) })