diff --git a/.github/workflows/update-competitive-matrix.yml b/.github/workflows/update-competitive-matrix.yml index b6e3355..a46338e 100644 --- a/.github/workflows/update-competitive-matrix.yml +++ b/.github/workflows/update-competitive-matrix.yml @@ -32,7 +32,7 @@ jobs: - name: Check for changes id: changes run: | - if git diff --quiet docs/index.html; then + if git diff --quiet docs/; then echo "changed=false" >> $GITHUB_OUTPUT else echo "changed=true" >> $GITHUB_OUTPUT @@ -45,11 +45,11 @@ jobs: git config user.email "github-actions[bot]@users.noreply.github.com" BRANCH="auto/competitive-matrix-$(date +%Y%m%d)" git checkout -b "$BRANCH" - git add docs/index.html - git commit -m "docs: update competitive matrix from latest competitor data" + git add docs/ + git commit -m "docs: update competitive matrix and migration pages from latest competitor data" git push -u origin "$BRANCH" gh pr create \ - --title "Update competitive matrix" \ + --title "Update competitive matrix and migration pages" \ --body-file /tmp/matrix-summary.md \ --base main env: diff --git a/scripts/update-competitive-matrix.ts b/scripts/update-competitive-matrix.ts index 8e50b5e..eced20b 100644 --- a/scripts/update-competitive-matrix.ts +++ b/scripts/update-competitive-matrix.ts @@ -4,8 +4,8 @@ * update-competitive-matrix.ts * * Fetches competitor READMEs from GitHub, extracts feature signals via keyword - * matching, and updates the comparison table in docs/index.html when evidence - * of new capabilities is found. + * matching, and updates the comparison table in docs/index.html and + * corresponding migration pages when evidence of new capabilities is found. * * Usage: * npx tsx scripts/update-competitive-matrix.ts # update in place @@ -13,7 +13,7 @@ * npx tsx scripts/update-competitive-matrix.ts --summary out.md # write markdown summary */ -import { readFileSync, writeFileSync } from "node:fs"; +import { readFileSync, writeFileSync, existsSync } from "node:fs"; import { resolve } from "node:path"; // ── Types ──────────────────────────────────────────────────────────────────── @@ -122,6 +122,14 @@ const FEATURE_RULES: FeatureRule[] = [ }, ]; +/** Maps competitor display names to their migration page paths (relative to docs/) */ +const COMPETITOR_MIGRATION_PAGES: Record = { + VidaiMock: "docs/migrate-from-vidaimock.html", + "mock-llm": "docs/migrate-from-mock-llm.html", + "piyook/llm-mock": "docs/migrate-from-piyook.html", + // MSW, Mokksy, Python don't have GitHub repos in COMPETITORS[] yet +}; + // ── Helpers ────────────────────────────────────────────────────────────────── const DRY_RUN = process.argv.includes("--dry-run"); @@ -174,6 +182,168 @@ function extractFeatures(text: string): Record { return result; } +/** + * Counts how many distinct LLM providers a competitor supports based on their + * README text. De-duplicates overlapping patterns (e.g. "anthropic" and "claude" + * both map to the same provider). + */ +function countProviders(text: string): number { + const lower = text.toLowerCase(); + + // Group patterns that refer to the same provider + const providerGroups: string[][] = [ + ["openai"], + ["claude", "anthropic"], + ["gemini", "google.*ai"], + ["bedrock", "aws"], + ["azure"], + ["vertex"], + ["ollama"], + ["cohere"], + ["mistral"], + ["groq"], + ["together"], + ["llama"], + ]; + + let count = 0; + for (const group of providerGroups) { + const found = group.some((kw) => new RegExp(kw, "i").test(lower)); + if (found) count++; + } + return count; +} + +// ── Migration Page Updating ───────────────────────────────────────────────── + +/** + * Updates a migration page's comparison table cells from the "no" state + * (✗) to the "yes" state (✓) when a feature is detected. + * + * Migration page tables use a different format than the index.html matrix: + * - "Yes" cells: ✓ + * - "No" cells: ✗ + * + * The function also updates numeric provider claims in both table cells and + * prose text (e.g., "5 providers" -> "8 providers"). + */ +function updateMigrationPage( + html: string, + competitorName: string, + features: Record, + providerCount: number, +): { html: string; changes: string[] } { + let result = html; + const changes: string[] = []; + + // Find the comparison table (class="comparison-table" or class="endpoint-table") + const tableMatch = result.match( + /([\s\S]*?)<\/table>/, + ); + if (!tableMatch) { + return { html: result, changes }; + } + + // Update feature cells: find rows where the competitor column shows ✗ + // and the feature was detected + for (const rule of FEATURE_RULES) { + if (!features[rule.rowLabel]) continue; + + // Migration tables have different row labels than the index matrix. + // We look for rows that conceptually match the feature rule. + // The competitor column is always the first data column (index 1) after the label. + const rowPatterns = buildMigrationRowPatterns(rule.rowLabel); + for (const rowPat of rowPatterns) { + const rowRegex = new RegExp( + `(\\s*\\s*)`, + ); + if (rowRegex.test(result)) { + result = result.replace(rowRegex, `$1`); + changes.push(`${competitorName}: ${rowPat} ✗ -> ✓`); + } + } + } + + // Update provider count claims in the competitor column of the table + // Match patterns like: >N providers<, >N+ providers< + if (providerCount > 0) { + result = updateProviderCounts(result, competitorName, providerCount, changes); + } + + return { html: result, changes }; +} + +/** + * Builds possible row label strings that a migration page might use for a given + * feature rule. Migration pages use more descriptive labels than the index matrix. + */ +function buildMigrationRowPatterns(rowLabel: string): string[] { + const patterns = [rowLabel]; + + // Add common migration-page variants + const variants: Record = { + "Chat Completions SSE": ["OpenAI Chat Completions", "Streaming SSE"], + "Responses API SSE": ["OpenAI Responses API"], + "Claude Messages API": ["Anthropic Claude"], + "Gemini streaming": ["Google Gemini"], + "WebSocket APIs": ["WebSocket protocols"], + "Structured output / JSON mode": ["Structured output / JSON mode", "Structured output"], + "Sequential / stateful responses": ["Sequential responses"], + "Docker image": ["Docker"], + "Fixture files (JSON)": ["Fixture files"], + "CLI server": ["CLI"], + "Error injection (one-shot)": ["Error injection"], + "Request journal": ["Request journal"], + "Drift detection": ["Drift detection"], + }; + + if (variants[rowLabel]) { + patterns.push(...variants[rowLabel]); + } + + return patterns; +} + +/** + * Scans the HTML for numeric provider claims and updates them if the detected + * count is higher. Handles patterns like: + * - "N providers" / "N+ providers" (in prose and table cells) + * - "supports N LLM" / "N LLM providers" + * - "N more providers" + */ +function updateProviderCounts( + html: string, + competitorName: string, + detectedCount: number, + changes: string[], +): string { + let result = html; + + // Pattern: N+ providers or N providers (in table cells and prose) + const providerCountRegex = /(\d+)\+?\s*providers/g; + result = result.replace(providerCountRegex, (match, numStr) => { + const currentCount = parseInt(numStr, 10); + if (detectedCount > currentCount) { + changes.push(`${competitorName}: provider count ${currentCount} -> ${detectedCount}`); + return `${detectedCount} providers`; + } + return match; + }); + + // Pattern: "supports N LLM" or "N LLM providers" + const llmProviderRegex = /(\d+)\+?\s*LLM\s*providers?/g; + result = result.replace(llmProviderRegex, (match, numStr) => { + const currentCount = parseInt(numStr, 10); + if (detectedCount > currentCount) { + changes.push(`${competitorName}: LLM provider count ${currentCount} -> ${detectedCount}`); + return `${detectedCount} LLM providers`; + } + return match; + }); + + return result; +} + // ── HTML Matrix Parsing & Updating ─────────────────────────────────────────── /** @@ -409,6 +579,8 @@ async function main(): Promise { // 1. Fetch competitor data const competitorFeatures = new Map>(); + const competitorProviderCounts = new Map(); + const competitorReadmes = new Map(); for (const comp of COMPETITORS) { console.log(`\n--- ${comp.name} (${comp.repo}) ---`); @@ -420,9 +592,14 @@ async function main(): Promise { } const combined = `${readme}\n${pkg}`; + competitorReadmes.set(comp.name, combined); const features = extractFeatures(combined); competitorFeatures.set(comp.name, features); + // Count providers + const provCount = countProviders(combined); + competitorProviderCounts.set(comp.name, provCount); + // Log detected features const detected = Object.entries(features) .filter(([, v]) => v) @@ -432,6 +609,9 @@ async function main(): Promise { } else { console.log(` No features detected from keywords.`); } + if (provCount > 0) { + console.log(` Detected ${provCount} LLM provider(s).`); + } } // 2. Read current HTML @@ -464,13 +644,53 @@ async function main(): Promise { if (DRY_RUN) { console.log("\n[DRY RUN] Would update docs/index.html with the above changes."); + console.log("[DRY RUN] Would also update migration pages for changed competitors."); return; } - // 5. Apply changes + // 5. Apply changes to index.html const updated = applyChanges(html, changes); writeFileSync(DOCS_PATH, updated, "utf-8"); console.log("\nUpdated docs/index.html successfully."); + + // 6. Update migration pages for competitors with changes + const docsDir = resolve(import.meta.dirname ?? __dirname, ".."); + const updatedCompetitors = new Set(changes.map((ch) => ch.competitor)); + + for (const compName of updatedCompetitors) { + const migrationPageRelPath = COMPETITOR_MIGRATION_PAGES[compName]; + if (!migrationPageRelPath) { + console.log(` No migration page mapped for ${compName}, skipping.`); + continue; + } + + const migrationPagePath = resolve(docsDir, migrationPageRelPath); + if (!existsSync(migrationPagePath)) { + console.log(` Migration page not found: ${migrationPagePath}, skipping.`); + continue; + } + + const migrationHtml = readFileSync(migrationPagePath, "utf-8"); + const features = competitorFeatures.get(compName) ?? {}; + const provCount = competitorProviderCounts.get(compName) ?? 0; + + const { html: updatedMigration, changes: migrationChanges } = updateMigrationPage( + migrationHtml, + compName, + features, + provCount, + ); + + if (migrationChanges.length > 0) { + writeFileSync(migrationPagePath, updatedMigration, "utf-8"); + console.log(`\nUpdated ${migrationPageRelPath}:`); + for (const ch of migrationChanges) { + console.log(` ${ch}`); + } + } else { + console.log(`\n${migrationPageRelPath}: no migration page changes needed.`); + } + } } main().catch((err) => { diff --git a/src/__tests__/competitive-matrix.test.ts b/src/__tests__/competitive-matrix.test.ts new file mode 100644 index 0000000..65894f6 --- /dev/null +++ b/src/__tests__/competitive-matrix.test.ts @@ -0,0 +1,461 @@ +import { describe, it, expect } from "vitest"; + +// ── Reimplement pure functions from scripts/update-competitive-matrix.ts ───── +// These mirror the logic so we can unit-test without requiring network access +// or dealing with import.meta.dirname in the test runner. + +// ── Provider count detection ──────────────────────────────────────────────── + +const PROVIDER_GROUPS: string[][] = [ + ["openai"], + ["claude", "anthropic"], + ["gemini", "google.*ai"], + ["bedrock", "aws"], + ["azure"], + ["vertex"], + ["ollama"], + ["cohere"], + ["mistral"], + ["groq"], + ["together"], + ["llama"], +]; + +function countProviders(text: string): number { + const lower = text.toLowerCase(); + let count = 0; + for (const group of PROVIDER_GROUPS) { + const found = group.some((kw) => new RegExp(kw, "i").test(lower)); + if (found) count++; + } + return count; +} + +// ── Feature rules (subset needed for tests) ──────────────────────────────── + +interface FeatureRule { + rowLabel: string; + keywords: string[]; +} + +const FEATURE_RULES: FeatureRule[] = [ + { + rowLabel: "Chat Completions SSE", + keywords: ["chat/completions", "streaming", "SSE", "server-sent", "stream.*true"], + }, + { + rowLabel: "WebSocket APIs", + keywords: ["websocket", "realtime", "ws://", "wss://"], + }, + { + rowLabel: "Embeddings API", + keywords: ["embedding", "/v1/embeddings", "embed"], + }, + { + rowLabel: "Structured output / JSON mode", + keywords: ["json_object", "json_schema", "structured output", "response_format"], + }, +]; + +function escapeRegex(str: string): string { + return str.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&"); +} + +// ── Migration page row pattern builder ────────────────────────────────────── + +function buildMigrationRowPatterns(rowLabel: string): string[] { + const patterns = [rowLabel]; + const variants: Record = { + "Chat Completions SSE": ["OpenAI Chat Completions", "Streaming SSE"], + "Responses API SSE": ["OpenAI Responses API"], + "Claude Messages API": ["Anthropic Claude"], + "Gemini streaming": ["Google Gemini"], + "WebSocket APIs": ["WebSocket protocols"], + "Structured output / JSON mode": ["Structured output / JSON mode", "Structured output"], + "Sequential / stateful responses": ["Sequential responses"], + "Docker image": ["Docker"], + "Fixture files (JSON)": ["Fixture files"], + "CLI server": ["CLI"], + "Error injection (one-shot)": ["Error injection"], + "Request journal": ["Request journal"], + "Drift detection": ["Drift detection"], + }; + if (variants[rowLabel]) { + patterns.push(...variants[rowLabel]); + } + return patterns; +} + +// ── Provider count update logic ───────────────────────────────────────────── + +function updateProviderCounts( + html: string, + competitorName: string, + detectedCount: number, + changes: string[], +): string { + let result = html; + + const providerCountRegex = /(\d+)\+?\s*providers/g; + result = result.replace(providerCountRegex, (match, numStr) => { + const currentCount = parseInt(numStr, 10); + if (detectedCount > currentCount) { + changes.push(`${competitorName}: provider count ${currentCount} -> ${detectedCount}`); + return `${detectedCount} providers`; + } + return match; + }); + + const llmProviderRegex = /(\d+)\+?\s*LLM\s*providers?/g; + result = result.replace(llmProviderRegex, (match, numStr) => { + const currentCount = parseInt(numStr, 10); + if (detectedCount > currentCount) { + changes.push(`${competitorName}: LLM provider count ${currentCount} -> ${detectedCount}`); + return `${detectedCount} LLM providers`; + } + return match; + }); + + return result; +} + +// ── Migration page update logic ───────────────────────────────────────────── + +function updateMigrationPage( + html: string, + competitorName: string, + features: Record, + providerCount: number, +): { html: string; changes: string[] } { + let result = html; + const changes: string[] = []; + + const tableMatch = result.match( + /
${escapeRegex(rowPat)}✗✓
([\s\S]*?)<\/table>/, + ); + if (!tableMatch) { + return { html: result, changes }; + } + + for (const rule of FEATURE_RULES) { + if (!features[rule.rowLabel]) continue; + + const rowPatterns = buildMigrationRowPatterns(rule.rowLabel); + for (const rowPat of rowPatterns) { + const rowRegex = new RegExp( + `(\\s*\\s*)`, + ); + if (rowRegex.test(result)) { + result = result.replace(rowRegex, `$1`); + changes.push(`${competitorName}: ${rowPat} ✗ -> ✓`); + } + } + } + + if (providerCount > 0) { + result = updateProviderCounts(result, competitorName, providerCount, changes); + } + + return { html: result, changes }; +} + +// ── Tests ─────────────────────────────────────────────────────────────────── + +describe("provider count extraction from README text", () => { + it("counts distinct providers from a README mentioning several", () => { + const readme = ` + Supports OpenAI, Anthropic Claude, Google Gemini, AWS Bedrock, + Azure OpenAI, and Cohere. + `; + expect(countProviders(readme)).toBe(6); + }); + + it("de-duplicates overlapping patterns (anthropic + claude = 1)", () => { + const readme = "Works with Anthropic and Claude models."; + expect(countProviders(readme)).toBe(1); + }); + + it("de-duplicates aws + bedrock as one provider", () => { + const readme = "Supports AWS Bedrock for model inference."; + expect(countProviders(readme)).toBe(1); + }); + + it("returns 0 for text with no provider mentions", () => { + expect(countProviders("This is a generic testing library.")).toBe(0); + }); + + it("counts all 12 provider groups when all are mentioned", () => { + const readme = ` + OpenAI, Claude, Gemini, Bedrock, Azure, Vertex AI, + Ollama, Cohere, Mistral, Groq, Together AI, Llama + `; + expect(countProviders(readme)).toBe(12); + }); + + it("is case-insensitive", () => { + expect(countProviders("OPENAI and ANTHROPIC")).toBe(2); + }); +}); + +describe("migration page table update logic", () => { + const SAMPLE_TABLE = ` +
${escapeRegex(rowPat)}✗✓
+ + + + + + + + + + + + + + + + + + + + + + + + +
CapabilityTestCompaimock
WebSocket protocols✗✓
Streaming SSE✓✓
Structured output✗✓
`; + + it("updates a No cell to Yes when the feature is detected", () => { + const features: Record = { + "Chat Completions SSE": false, + "WebSocket APIs": true, + "Embeddings API": false, + "Structured output / JSON mode": false, + }; + + const { html, changes } = updateMigrationPage(SAMPLE_TABLE, "TestComp", features, 0); + + // WebSocket protocols row should now show checkmark + expect(html).toContain( + 'WebSocket protocols\n ✓', + ); + expect(changes.length).toBeGreaterThan(0); + expect(changes[0]).toContain("WebSocket protocols"); + }); + + it("does not downgrade an already-yes cell", () => { + const features: Record = { + "Chat Completions SSE": true, // maps to "Streaming SSE" variant + "WebSocket APIs": false, + "Embeddings API": false, + "Structured output / JSON mode": false, + }; + + const { html } = updateMigrationPage(SAMPLE_TABLE, "TestComp", features, 0); + + // Streaming SSE was already ✓, should remain unchanged + expect(html).toContain( + 'Streaming SSE\n ✓', + ); + }); + + it("returns no changes when no table is found", () => { + const noTableHtml = "

No table here

"; + const features: Record = { + "WebSocket APIs": true, + "Chat Completions SSE": false, + "Embeddings API": false, + "Structured output / JSON mode": false, + }; + + const { html, changes } = updateMigrationPage(noTableHtml, "TestComp", features, 5); + + expect(html).toBe(noTableHtml); + expect(changes).toHaveLength(0); + }); + + it("handles endpoint-table class as well as comparison-table", () => { + const endpointTable = SAMPLE_TABLE.replace("comparison-table", "endpoint-table"); + const features: Record = { + "Chat Completions SSE": false, + "WebSocket APIs": true, + "Embeddings API": false, + "Structured output / JSON mode": false, + }; + + const { changes } = updateMigrationPage(endpointTable, "TestComp", features, 0); + + expect(changes.length).toBeGreaterThan(0); + }); + + it("updates multiple features in one pass", () => { + const features: Record = { + "Chat Completions SSE": false, + "WebSocket APIs": true, + "Embeddings API": false, + "Structured output / JSON mode": true, + }; + + const { html, changes } = updateMigrationPage(SAMPLE_TABLE, "TestComp", features, 0); + + // Both WebSocket protocols and Structured output should be updated + expect(changes.length).toBe(2); + expect(html).not.toContain("✗"); + }); +}); + +describe("numeric provider claim updates", () => { + it('updates "5 providers" to "8 providers" when detected count is higher', () => { + const html = "

Supports 5 providers out of the box.

"; + const changes: string[] = []; + + const result = updateProviderCounts(html, "TestComp", 8, changes); + + expect(result).toContain("8 providers"); + expect(result).not.toContain("5 providers"); + expect(changes.length).toBe(1); + }); + + it('updates "5+ providers" to "8 providers" (strips the +)', () => { + const html = "5+ providers"; + const changes: string[] = []; + + const result = updateProviderCounts(html, "TestComp", 8, changes); + + expect(result).toContain("8 providers"); + expect(result).not.toContain("5+"); + }); + + it("does not update when detected count is lower or equal", () => { + const html = "

Supports 10 providers.

"; + const changes: string[] = []; + + const result = updateProviderCounts(html, "TestComp", 8, changes); + + expect(result).toContain("10 providers"); + expect(changes).toHaveLength(0); + }); + + it("updates N LLM providers pattern", () => { + const html = "

supports 3 LLM providers

"; + const changes: string[] = []; + + const result = updateProviderCounts(html, "TestComp", 7, changes); + + expect(result).toContain("7 LLM providers"); + expect(changes.length).toBe(1); + }); + + it("handles no numeric claims gracefully", () => { + const html = "

A great testing tool.

"; + const changes: string[] = []; + + const result = updateProviderCounts(html, "TestComp", 5, changes); + + expect(result).toBe(html); + expect(changes).toHaveLength(0); + }); + + it("handles multiple provider count references in one document", () => { + const html = ` +

Supports 5 providers including OpenAI.

+ 5+ providers + `; + const changes: string[] = []; + + const result = updateProviderCounts(html, "TestComp", 9, changes); + + // Both occurrences should be updated + expect(result).not.toContain("5 providers"); + expect(result).not.toContain("5+"); + expect((result.match(/9 providers/g) || []).length).toBe(2); + }); + + it("does not change provider count when equal", () => { + const html = "8 providers"; + const changes: string[] = []; + + const result = updateProviderCounts(html, "TestComp", 8, changes); + + expect(result).toBe(html); + expect(changes).toHaveLength(0); + }); +}); + +describe("migration page update with provider counts", () => { + const PAGE_WITH_COUNTS = ` +

TestComp supports 5 providers today.

+ + + + + + + + + + + + + + + + + + + + +
CapabilityTestCompaimock
LLM providers5+10+
WebSocket protocols✗✓
`; + + it("updates both feature cells and provider counts in one call", () => { + const features: Record = { + "Chat Completions SSE": false, + "WebSocket APIs": true, + "Embeddings API": false, + "Structured output / JSON mode": false, + }; + + const { html, changes } = updateMigrationPage(PAGE_WITH_COUNTS, "TestComp", features, 8); + + // Feature cell should be updated + expect(html).not.toContain("✗"); + // Provider count in prose should be updated + expect(html).toContain("8 providers"); + expect(changes.length).toBeGreaterThanOrEqual(2); + }); + + it("leaves provider count alone when detected is not higher", () => { + const features: Record = { + "Chat Completions SSE": false, + "WebSocket APIs": false, + "Embeddings API": false, + "Structured output / JSON mode": false, + }; + + const { html, changes } = updateMigrationPage(PAGE_WITH_COUNTS, "TestComp", features, 3); + + // Count should remain as-is + expect(html).toContain("5 providers"); + expect(changes).toHaveLength(0); + }); +}); + +describe("buildMigrationRowPatterns", () => { + it("returns the original label plus variants", () => { + const patterns = buildMigrationRowPatterns("WebSocket APIs"); + expect(patterns).toContain("WebSocket APIs"); + expect(patterns).toContain("WebSocket protocols"); + }); + + it("returns just the label for unknown rules", () => { + const patterns = buildMigrationRowPatterns("Some Unknown Feature"); + expect(patterns).toEqual(["Some Unknown Feature"]); + }); + + it("returns multiple variants for Chat Completions SSE", () => { + const patterns = buildMigrationRowPatterns("Chat Completions SSE"); + expect(patterns).toContain("OpenAI Chat Completions"); + expect(patterns).toContain("Streaming SSE"); + }); +});