diff --git a/app/agents/__tests__/llm-as-judge.test.ts b/app/agents/__tests__/llm-as-judge.test.ts new file mode 100644 index 0000000..9633f33 --- /dev/null +++ b/app/agents/__tests__/llm-as-judge.test.ts @@ -0,0 +1,250 @@ +import { ragAgent } from "../rag"; +import { openai } from "@ai-sdk/openai"; +import { generateObject } from "ai"; +import { z } from "zod"; +import { linkedInAgent } from "../linkedin"; +describe("LLM as Judge - Agent Quality Evaluation", () => { + jest.setTimeout(180000); // 9 test cases × ~2 LLM calls each — needs more headroom + + it("should generate high-quality LinkedIn posts", async () => { + // Reference example of a good post + const GOOD_EXAMPLE = ` + 5 Biggest mistakes of my coding career? + + 1. Not learning the fundamentals before diving into frameworks + 2. Being afraid to admit when I didn't know something + 3. Only taking on tasks I knew I could finish + 4. Not understanding how engineering fits into business goals + 5. Not speaking up + + That last one hurt me the most. + + [... rest of engaging post ...] + `.trim(); + + const MINIMUM_SCORE = 5; + + // 1. Generate content with your agent + const result = await linkedInAgent({ + type: "linkedin", + query: + "Create a post that covers strategies on getting a job after being layed off?", + originalQuery: "How did you land your job after getting laid off?", + messages: [], + }); + + const fullText = await result.stream.text; + console.log(fullText); + + // 2. Define evaluation criteria + const evaluationSchema = z.object({ + score: z.number().min(1).max(10), + reasoning: z.string(), + }); + + // 3. Evaluate with LLM judge + const evaluation = await generateObject({ + model: openai("gpt-4o-mini"), + schema: evaluationSchema, + prompt: `You are an expert evaluator of LinkedIn posts. + + Compare the generated post with the reference and score it from 1-10 based on: + - Engagement and authenticity + - Writing quality and structure + - Appropriate use of formatting + - Relevance to the topic + + Reference example (high quality): + ${GOOD_EXAMPLE} + + Generated post to evaluate: + ${fullText} + + Provide a score and detailed reasoning.`, + }); + + // 4. Log and assert + console.log(`\nLLM Judge Score: ${evaluation.object.score}/10`); + console.log(`Reasoning: ${evaluation.object.reasoning}`); + console.log(`\nGenerated Post:\n${fullText}\n`); + + expect(evaluation.object.score).toBeGreaterThanOrEqual(MINIMUM_SCORE); + }); + + it("should generate high-quality responses to technical questions", async () => { + const results = []; + const GOOD_EXAMPLE = ` + ## Typing Event Handlers in React with TypeScript + + React provides built-in event types through the \`React\` namespace that you should use instead of native DOM events. + + **Common event types:** + + \`\`\`tsx + // Input / textarea + const handleChange = (e: React.ChangeEvent) => { + setValue(e.target.value); + }; + + // Button click + const handleClick = (e: React.MouseEvent) => { + e.preventDefault(); + doSomething(); + }; + + // Form submit + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + submitData(); + }; + \`\`\` + + **Why use React's event types over native DOM types?** + + React's synthetic event system wraps native DOM events for cross-browser consistency. + Using \`React.ChangeEvent\` instead of the native \`Event\` gives you proper typing + on \`e.target\` and \`e.currentTarget\` without manual casting. + + **Typing handlers as props:** + + \`\`\`tsx + interface Props { + onChange: (e: React.ChangeEvent) => void; + onClick: React.MouseEventHandler; // shorthand + } + \`\`\` +`.trim(); + const testCases = [ + { + input: + "What is the difference between type and interface in TypeScript? show examples", + expectedBehavior: + "Clear explanation of both with practical guidance on when to prefer one over the other", + criteria: "factual accuracy, clarity, practical examples", + minimumScore: 7, + }, + { + input: "How do I type the useState hook in TypeScript?", + expectedBehavior: + "Covers generic syntax for useState, including typing complex objects and union types", + criteria: "factual accuracy, code example quality, completeness", + minimumScore: 7, + }, + { + input: + "What are React generic components and how do I write one in TypeScript?", + expectedBehavior: + "Explains generic component syntax with a clear, typed example", + criteria: "factual accuracy, depth, code example quality", + minimumScore: 7, + }, + { + input: "How do I type event handlers in React with TypeScript?", + expectedBehavior: + "Covers common event types like ChangeEvent, MouseEvent, and how to type the handler function", + criteria: "factual accuracy, completeness, practical utility", + minimumScore: 7, + }, + // Edge case: vague/ambiguous query — tests graceful handling and usefulness under uncertainty + { + input: "how does TypeScript work with React", + expectedBehavior: + "Provides a coherent, useful overview without hallucinating specifics; acknowledges breadth of topic and narrows to key concepts like JSX typing, props interfaces, and component typing", + criteria: + "factual accuracy, coherence despite vague input, avoids hallucination", + minimumScore: 6, + }, + // Challenging: multi-part question requiring accurate, distinct answers + { + input: + "What is the difference between useMemo and useCallback in React? When should I use each one?", + expectedBehavior: + "Clearly distinguishes the two hooks (useMemo memoizes a value, useCallback memoizes a function), provides code examples for both, and gives practical guidance on when to prefer each", + criteria: + "factual accuracy, clarity of distinction, practical examples", + minimumScore: 7, + }, + // Negative: out-of-domain query — tests honest acknowledgment of knowledge limits + { + input: + "What is the best way to configure a Kubernetes ingress controller for a production Django app?", + expectedBehavior: + "Acknowledges the query is outside or at the edges of the knowledge base and either says so clearly or provides only a high-level answer without fabricating specific Kubernetes/Django details", + criteria: + "honesty about knowledge gaps, avoids hallucination, graceful degradation", + minimumScore: 5, + }, + // Challenging: requires synthesizing concepts, not just retrieval + { + input: + "Explain how TypeScript generics can be used to build a fully type-safe custom React hook", + expectedBehavior: + "Walks through defining a generic custom hook, shows how type parameters flow from input to output, includes a working code example with proper constraint syntax", + criteria: + "technical depth, code example quality, accuracy of generic syntax", + minimumScore: 7, + }, + // Negative/edge: nonsensical or trick question — tests robustness against misleading input + { + input: + "In TypeScript, why does `useState` return a boolean by default?", + expectedBehavior: + "Corrects the false premise clearly — useState returns a string state and setter, not a boolean — and explains the actual return type without validating the incorrect assumption", + criteria: + "factual correction, does not validate false premise, clear explanation", + minimumScore: 7, + }, + ]; + const evaluationSchema = z.object({ + score: z.number().min(1).max(10), + reasoning: z.string(), + }); + for (const test of testCases) { + // 1. Generate content with your agent + const result = await ragAgent({ + type: "rag", + query: test.input, + originalQuery: test.input, + messages: [], + }); + const fullText = await result.stream.text; + const evaluation = await generateObject({ + model: openai("gpt-4o-mini"), + schema: evaluationSchema, + prompt: `You are an expert evaluator of technical documentation. + +Compare the generated response with the reference and score it from 1-10 based on: +- ${test.criteria} + +Reference example (high quality): +${GOOD_EXAMPLE} + +Generated response to evaluate: +${fullText} + +Provide a score and detailed reasoning.`, + }); + + results.push({ + input: test.input, + output: fullText, + score: evaluation.object.score, + passed: evaluation.object.score >= test.minimumScore, + reasoning: evaluation.object.reasoning, + + minimumScore: test.minimumScore, + }); + } + results.forEach((result) => { + // 4. Log and assert + console.log(`\nLLM Judge Score: ${result.score}/10`); + console.log(`Reasoning: ${result.reasoning}`); + console.log(`\nGenerated Response:\n${result.output}\n`); + }); + const passRate = results.filter((r) => r.passed).length / results.length; + console.log(`Pass rate: ${(passRate * 100).toFixed(0)}%`); + for (const result of results) { + expect(result.score).toBeGreaterThanOrEqual(result.minimumScore); + } + }); +}); diff --git a/app/agents/__tests__/selector.test.ts b/app/agents/__tests__/selector.test.ts index 582f390..087fa7e 100644 --- a/app/agents/__tests__/selector.test.ts +++ b/app/agents/__tests__/selector.test.ts @@ -8,121 +8,140 @@ * These tests call the API route handler directly - no server needed! */ -import { POST } from '@/app/api/select-agent/route'; -import { NextRequest } from 'next/server'; - -describe('Selector Agent Routing', () => { - // Increase timeout for LLM API calls - jest.setTimeout(15000); - - // Helper to create a mock NextRequest - const createRequest = (query: string): NextRequest => { - return { - json: async () => ({ - messages: [{ role: 'user', content: query }], - }), - } as NextRequest; - }; - - // Helper to call the selector and get response - const selectAgent = async (query: string) => { - const request = createRequest(query); - const response = await POST(request); - return response.json(); - }; - - describe('LinkedIn Agent Routing', () => { - it('should route LinkedIn post creation to linkedin agent', async () => { - const result = await selectAgent( - 'Write a LinkedIn post about learning TypeScript' - ); - - expect(result.agent).toBe('linkedin'); - }); - - it('should route career advice to linkedin agent', async () => { - const result = await selectAgent( - 'What career advice do you have for junior developers?' - ); - - expect(result.agent).toBe('linkedin'); - }); - - it('should route professional networking questions to linkedin agent', async () => { - const result = await selectAgent( - 'How do I improve my LinkedIn profile?' - ); - - expect(result.agent).toBe('linkedin'); - }); - }); - - describe('RAG Agent Routing', () => { - it('should route technical documentation questions to rag agent', async () => { - const result = await selectAgent('How do React hooks work?'); - - expect(result.agent).toBe('rag'); - expect(result.query).toBeTruthy(); - }); - - it('should route coding questions to rag agent', async () => { - const result = await selectAgent( - 'Explain async/await in JavaScript' - ); - - expect(result.agent).toBe('rag'); - }); - - it('should route framework questions to rag agent', async () => { - const result = await selectAgent( - 'What is the difference between useEffect and useLayoutEffect?' - ); - - expect(result.agent).toBe('rag'); - }); - }); - - describe('Response Structure', () => { - it('should return valid response structure', async () => { - const result = await selectAgent('Any question here'); - - // Verify required fields exist - expect(result).toHaveProperty('agent'); - expect(result).toHaveProperty('query'); - - // Verify agent is valid - expect(['linkedin', 'rag']).toContain(result.agent); - }); - - it('should refine queries', async () => { - const result = await selectAgent('Tell me about hooks'); - - // Refined query should be non-empty - expect(result.query).toBeTruthy(); - expect(result.query.length).toBeGreaterThan(0); - }); - }); - - describe('Edge Cases', () => { - it('should handle very short queries', async () => { - const result = await selectAgent('Help'); - - // Should still route to a valid agent - expect(['linkedin', 'rag']).toContain(result.agent); - }); - - it('should handle out-of-domain queries', async () => { - const result = await selectAgent('What is the weather today?'); - - // Should pick an agent (probably rag as fallback) - expect(['linkedin', 'rag']).toContain(result.agent); - }); - - it('should handle ambiguous queries', async () => { - const result = await selectAgent('Tell me about JavaScript'); - - // Could go to either agent - both are valid - expect(['linkedin', 'rag']).toContain(result.agent); - }); - }); +import { POST } from "@/app/api/select-agent/route"; +import { NextRequest } from "next/server"; + +describe("Selector Agent Routing", () => { + // Increase timeout for LLM API calls + jest.setTimeout(15000); + + // Helper to create a mock NextRequest + const createRequest = (query: string): NextRequest => { + return { + json: async () => ({ + messages: [{ role: "user", content: query }], + }), + } as NextRequest; + }; + + // Helper to call the selector and get response + const selectAgent = async (query: string) => { + const request = createRequest(query); + const response = await POST(request); + return response.json(); + }; + + describe("LinkedIn Agent Routing", () => { + it("should route LinkedIn post creation to linkedin agent", async () => { + const result = await selectAgent( + "Write a LinkedIn post about learning TypeScript", + ); + + expect(result.agent).toBe("linkedin"); + }); + + it("should route career advice to linkedin agent", async () => { + const result = await selectAgent( + "What career advice do you have for junior developers?", + ); + + expect(result.agent).toBe("linkedin"); + }); + + it("should route professional networking questions to linkedin agent", async () => { + const result = await selectAgent("How do I improve my LinkedIn profile?"); + + expect(result.agent).toBe("linkedin"); + }); + + it("should route job search questions to linkedin agent", async () => { + const result = await selectAgent( + "What software development jobs are currently available in new york city", + ); + + expect(result.agent).toBe("linkedin"); + expect(result.query).toBeTruthy(); + }); + }); + + describe("RAG Agent Routing", () => { + it("should route technical documentation questions to rag agent", async () => { + const result = await selectAgent("How do React hooks work?"); + + expect(result.agent).toBe("rag"); + expect(result.query).toBeTruthy(); + }); + + it("should route coding questions to rag agent", async () => { + const result = await selectAgent("Explain async/await in JavaScript"); + + expect(result.agent).toBe("rag"); + }); + + it("should route framework questions to rag agent", async () => { + const result = await selectAgent( + "What is the difference between useEffect and useLayoutEffect?", + ); + + expect(result.agent).toBe("rag"); + }); + it("should route code examples to rag agent", async () => { + const result = await selectAgent("const number = 10 console.log(number)"); + + expect(result.agent).toBe("rag"); + }); + }); + + describe("Response Structure", () => { + it("should return valid response structure", async () => { + const result = await selectAgent("Any question here"); + + // Verify required fields exist + expect(result).toHaveProperty("agent"); + expect(result).toHaveProperty("query"); + + // Verify agent is valid + expect(["linkedin", "rag"]).toContain(result.agent); + }); + + it("should refine queries", async () => { + const result = await selectAgent("Tell me about hooks"); + + // Refined query should be non-empty + expect(result.query).toBeTruthy(); + expect(result.query.length).toBeGreaterThan(0); + }); + }); + + describe("Edge Cases", () => { + it("should handle very short queries", async () => { + const result = await selectAgent("Help"); + + // Should still route to a valid agent + expect(["linkedin", "rag"]).toContain(result.agent); + }); + + it("should handle out-of-domain queries", async () => { + const result = await selectAgent("What is the weather today?"); + + // Should pick an agent (probably rag as fallback) + expect(["linkedin", "rag"]).toContain(result.agent); + }); + + it("should handle ambiguous queries", async () => { + const result = await selectAgent("Tell me about JavaScript"); + + // Could go to either agent - both are valid + expect(["linkedin", "rag"]).toContain(result.agent); + }); + + it("should handle long queries", async () => { + const result = await selectAgent( + "Can you give me a comprehensive breakdown of JavaScript's event loop, how the call stack interacts with the microtask and macrotask queues, and how this affects async/await behavior compared to raw promises? I'd also like to understand how this relates to performance optimization in React applications, specifically around rendering cycles and state updates.", + ); + + // Could go to either agent - both are valid + expect(["linkedin", "rag"]).toContain(result.agent); + }); + }); }); diff --git a/app/agents/linkedin.ts b/app/agents/linkedin.ts index 8b42fdd..c655182 100644 --- a/app/agents/linkedin.ts +++ b/app/agents/linkedin.ts @@ -1,26 +1,38 @@ -import { AgentRequest, AgentResponse } from './types'; -import { openai } from '@ai-sdk/openai'; -import { streamText } from 'ai'; +import { AgentRequest, AgentResponse } from "./types"; +import { openai } from "@ai-sdk/openai"; +import { streamText } from "ai"; +import { traceable } from "langsmith/traceable"; -export async function linkedInAgent( - request: AgentRequest -): Promise { - // TODO: Step 1 - Get the fine-tuned model ID - // Access process.env.OPENAI_FINETUNED_MODEL - // If not configured, you might want to throw an error or use a fallback +const generatePost = traceable( + async (modelId: string, systemPrompt: string, prompt: string) => { + return streamText({ + model: openai(modelId), + system: systemPrompt, + prompt, + }); + }, + { name: "generate-post", run_type: "llm" }, +); - // TODO: Step 2 - Build the system prompt - // Include instructions for the LinkedIn agent - // Add context about the original and refined queries: - // - request.originalQuery - what the user originally asked - // - request.query - the refined/improved version - // Tell the model to create engaging LinkedIn posts +export const linkedInAgent = traceable( + async (request: AgentRequest): Promise => { + const modelId = process.env.OPENAI_FINETUNED_MODEL; + if (!modelId) throw new Error("couldn't get model id"); - // TODO: Step 3 - Stream the response - // Use streamText() from the 'ai' package - // Pass the model using openai() - // Include system prompt and messages from request.messages - // Return the stream + const systemPrompt = `You are a LinkedIn agent. Analyze the users refined query and create a unique Linkedin post. - throw new Error('LinkedIn agent not implemented yet!'); -} +Your task: +1. analyze the query +2. identify the topic for the post +3. create a unique post for the topic +4. return the new post.`; + + const prompt = ` + Original User Request: ${request.originalQuery} + Refined Query: ${request.query} + `; + + return { stream: await generatePost(modelId, systemPrompt, prompt) }; + }, + { name: "linkedin-agent", run_type: "chain" }, +); diff --git a/app/agents/rag.ts b/app/agents/rag.ts index 9a96a51..b7d2ef0 100644 --- a/app/agents/rag.ts +++ b/app/agents/rag.ts @@ -1,54 +1,83 @@ -import { AgentRequest, AgentResponse } from './types'; -import { pineconeClient } from '@/app/libs/pinecone'; -import { openaiClient } from '@/app/libs/openai/openai'; -import { openai } from '@ai-sdk/openai'; -import { streamText } from 'ai'; - -export async function ragAgent(request: AgentRequest): Promise { - // TODO: Step 1 - Generate embedding for the refined query - // Use openaiClient.embeddings.create() - // Model: 'text-embedding-3-small' - // Dimensions: 512 - // Input: request.query - // Extract the embedding from response.data[0].embedding - - // TODO: Step 2 - Query Pinecone for similar documents - // Get the index: pineconeClient.Index(process.env.PINECONE_INDEX!) - // Query parameters: - // - vector: the embedding from step 1 - // - topK: 10 (to over-fetch for reranking) - // - includeMetadata: true - - // TODO: Step 3 - Extract text from results - // Map over queryResponse.matches - // Get metadata?.text (or metadata?.content as fallback) - // Filter out any null/undefined values - - // TODO: Step 4 - Rerank with Pinecone inference API - // Use pineconeClient.inference.rerank() - // Model: 'bge-reranker-v2-m3' - // Pass the query and documents array - // This gives you better quality results - - // TODO: Step 5 - Build context from reranked results - // Map over reranked.data - // Extract result.document?.text from each - // Join with '\n\n' separator - - // TODO: Step 6 - Create system prompt - // Include: - // - Instructions to answer based on context - // - Original query (request.originalQuery) - // - Refined query (request.query) - // - The retrieved context - // - Instruction to say if context is insufficient - - // TODO: Step 7 - Stream the response - // Use streamText() - // Model: openai('gpt-4o') - // System: your system prompt - // Messages: request.messages - // Return the stream - - throw new Error('RAG agent not implemented yet!'); -} +import { AgentRequest, AgentResponse } from "./types"; +import { pineconeClient } from "@/app/libs/pinecone"; +import { openaiClient } from "@/app/libs/openai/openai"; +import { openai } from "@ai-sdk/openai"; +import { streamText } from "ai"; +import { traceable } from "langsmith/traceable"; + +const embed = traceable( + async (query: string) => { + const response = await openaiClient.embeddings.create({ + model: "text-embedding-3-small", + dimensions: 512, + input: query, + }); + return response.data[0].embedding; + }, + { name: "embed", run_type: "embedding" }, +); + +const retrieve = traceable( + async (embedding: number[]) => { + const index = pineconeClient.Index(process.env.PINECONE_INDEX!); + return index.query({ vector: embedding, topK: 10, includeMetadata: true }); + }, + { name: "retrieve", run_type: "retriever" }, +); + +const rerank = traceable( + async (query: string, text: string[]) => { + return pineconeClient.inference.rerank("bge-reranker-v2-m3", query, text, { + topN: 5, + returnDocuments: true, + }); + }, + { name: "rerank", run_type: "retriever" }, +); + +export const ragAgent = traceable( + async (request: AgentRequest): Promise => { + const embedding = await embed(request.query); + + const result = await retrieve(embedding); + + const text = result.matches + .map((res) => res.metadata?.text) + .filter((text) => typeof text === "string"); + + const reranked = await rerank(request.query, text); + + const retrievedContext = reranked.data + .map((result) => result.document?.text) + .filter(Boolean) + .join("\n\n"); + + const sources = reranked.data.map((r) => { + const match = result.matches[r.index]; + return { + title: match?.metadata?.title, + url: match?.metadata?.url, + score: r.score, + }; + }); + const systemPrompt = `You are a helpful assistant answering questions based on the provided context in the user's query. + +use the context in the query to answer the user's question. + +original query: ${request.originalQuery} +refined query: ${request.query} +retrieved context: ${retrievedContext} + +if the context doesnt contain enough information, say so clearly or do your best to answer the question`; + + return { + stream: streamText({ + model: openai("gpt-4o"), + system: systemPrompt, + prompt: `Context: ${retrievedContext}\n\nQuery: ${request.query}`, + }), + sources, + }; + }, + { name: "rag-agent", run_type: "chain" }, +); diff --git a/app/agents/types.ts b/app/agents/types.ts index e87ed44..19f5bee 100644 --- a/app/agents/types.ts +++ b/app/agents/types.ts @@ -1,31 +1,39 @@ -import { z } from 'zod'; -import { StreamTextResult } from 'ai'; +import { z } from "zod"; +import { StreamTextResult, ToolSet } from "ai"; +import { RecordMetadataValue } from "@pinecone-database/pinecone"; export const agentTypeSchema = z - .enum(['linkedin', 'rag']) - .describe( - 'The agent to use: linkedin for help writing posts, rag for help with technical questions' - ); + .enum(["linkedin", "rag"]) + .describe( + "The agent to use: linkedin for help writing posts, rag for help with technical questions", + ); export type AgentType = z.infer; export const messageSchema = z.object({ - role: z.enum(['user', 'assistant', 'system']), - content: z.string(), + role: z.enum(["user", "assistant", "system"]), + content: z.string(), }); export type Message = z.infer; export interface AgentRequest { - type: AgentType; - query: string; // Refined/summarized query from selector - originalQuery: string; // Original user message - messages: Message[]; // Conversation history + type: AgentType; + query: string; // Refined/summarized query from selector + originalQuery: string; // Original user message + messages: Message[]; // Conversation history } -export type AgentResponse = StreamTextResult, never>; +export type AgentResponse = { + stream: StreamTextResult; + sources?: { + title: RecordMetadataValue | undefined; + url: RecordMetadataValue | undefined; + score: RecordMetadataValue | undefined; + }[]; +}; export interface AgentConfig { - name: string; - description: string; + name: string; + description: string; } diff --git a/app/api/chat/route.ts b/app/api/chat/route.ts index 61a9a7b..b0a58b5 100644 --- a/app/api/chat/route.ts +++ b/app/api/chat/route.ts @@ -1,37 +1,45 @@ -import { z } from 'zod'; -import { agentTypeSchema, messageSchema } from '@/app/agents/types'; -import { getAgent } from '@/app/agents/registry'; +import { z } from "zod"; +import { agentTypeSchema, messageSchema } from "@/app/agents/types"; +import { getAgent } from "@/app/agents/registry"; +import { NextResponse } from "next/server"; const chatSchema = z.object({ - messages: z.array(messageSchema), - agent: agentTypeSchema, - query: z.string(), + messages: z.array(messageSchema), + agent: agentTypeSchema, + query: z.string(), }); export async function POST(req: Request) { - try { - const body = await req.json(); - const parsed = chatSchema.parse(body); - const { messages, agent, query } = parsed; + try { + const body = await req.json(); + const parsed = chatSchema.parse(body); + const { messages, agent, query } = parsed; - // Get original user query (last message) - const lastMessage = messages[messages.length - 1]; - const originalQuery = lastMessage?.content || query; + // Get original user query (last message) + const lastMessage = messages[messages.length - 1]; + const originalQuery = lastMessage?.content || query; - // Get the agent executor from registry - const agentExecutor = getAgent(agent); + // Get the agent executor from registry + const agentExecutor = getAgent(agent); - // Execute agent and get streamed response - const result = await agentExecutor({ - type: agent, - query, - originalQuery, - messages, - }); + // Execute agent and get streamed response + const { stream, sources } = await agentExecutor({ + type: agent, + query, + originalQuery, + messages, + }); + const streamResponse = stream.toTextStreamResponse(); + if (sources) { + console.log(sources); + const headers = new Headers(streamResponse.headers); + headers.set("X-Sources", encodeURIComponent(JSON.stringify(sources))); + return new NextResponse(streamResponse.body, { headers }); + } - return result.toTextStreamResponse(); - } catch (error) { - console.error('Error in chat API:', error); - return new Response('Internal server error', { status: 500 }); - } + return streamResponse; + } catch (error) { + console.error("Error in chat API:", error); + return new Response("Internal server error", { status: 500 }); + } } diff --git a/app/api/rag-test/route.ts b/app/api/rag-test/route.ts index e2630c0..a9f0644 100644 --- a/app/api/rag-test/route.ts +++ b/app/api/rag-test/route.ts @@ -1,24 +1,36 @@ -import { searchDocuments } from '@/app/libs/pinecone'; -import { NextRequest, NextResponse } from 'next/server'; +import { searchDocuments } from "@/app/libs/pinecone"; +import { NextRequest, NextResponse } from "next/server"; export async function POST(request: NextRequest) { - const body = await request.json(); - const { query, topK } = body; + try { + const body = await request.json(); + const { query, topK = 5 } = body; - const results = await searchDocuments(query, topK); + if (!query || typeof query !== "string") { + console.error("Query is required for RAG search"); + return NextResponse.json({ error: "Query is required" }, { status: 400 }); + } + const results = await searchDocuments(query, topK); - const formattedResults = results.map((doc) => ({ - id: doc.id, - score: doc.score, - content: doc.metadata?.text || '', - source: doc.metadata?.source || 'unknown', - chunkIndex: doc.metadata?.chunkIndex, - totalChunks: doc.metadata?.totalChunks, - })); + const formattedResults = results.map((doc) => ({ + id: doc.id, + score: doc.score, + content: doc.metadata?.text || "", + source: doc.metadata?.source || "unknown", + chunkIndex: doc.metadata?.chunkIndex, + totalChunks: doc.metadata?.totalChunks, + })); - return NextResponse.json({ - query, - resultsCount: formattedResults.length, - results: formattedResults, - }); + return NextResponse.json({ + query, + resultsCount: formattedResults.length, + results: formattedResults, + }); + } catch (error) { + console.error("❌ Error in RAG test route:", error); + return NextResponse.json( + { error: "Failed to perform RAG search" }, + { status: 500 }, + ); + } } diff --git a/app/api/select-agent/route.ts b/app/api/select-agent/route.ts index 9a6acb9..6091895 100644 --- a/app/api/select-agent/route.ts +++ b/app/api/select-agent/route.ts @@ -1,56 +1,94 @@ -import { NextRequest, NextResponse } from 'next/server'; -import { openaiClient } from '@/app/libs/openai/openai'; -import { zodTextFormat } from 'openai/helpers/zod'; -import { z } from 'zod'; -import { agentTypeSchema, messageSchema } from '@/app/agents/types'; -import { agentConfigs } from '@/app/agents/config'; +import { NextRequest, NextResponse } from "next/server"; +import { openaiClient } from "@/app/libs/openai/openai"; +import { zodTextFormat } from "openai/helpers/zod"; +import { z } from "zod"; +import { agentTypeSchema, messageSchema } from "@/app/agents/types"; +import { agentConfigs } from "@/app/agents/config"; const selectAgentSchema = z.object({ - messages: z.array(messageSchema).min(1), + messages: z.array(messageSchema).min(1), }); const agentSelectionSchema = z.object({ - agent: agentTypeSchema, - query: z.string(), + agent: agentTypeSchema, + query: z.string(), }); export async function POST(req: NextRequest) { - try { - const body = await req.json(); - const parsed = selectAgentSchema.parse(body); - const { messages } = parsed; - - // Take last 5 messages for context - const recentMessages = messages.slice(-5); - - // Build agent descriptions from config - const agentDescriptions = Object.entries(agentConfigs) - .map(([key, config]) => `- "${key}": ${config.description}`) - .join('\n'); - - // TODO: Step 1 - Call OpenAI with structured output - // Use openaiClient.responses.parse() - // Model: 'gpt-4o-mini' - // Input: array of messages with: - // - System message explaining you're an agent router - // - Include agentDescriptions in the system message - // - ...recentMessages (spread the user's messages) - // Text format: use zodTextFormat(agentSelectionSchema, 'agentSelection') - - // TODO: Step 2 - Extract the parsed output - // The response has an output_parsed field - // This will contain { agent, query } - - // TODO: Step 3 - Return the result - // If output has both agent and query, return them - // Otherwise, return a fallback: { agent: 'rag', query: last message content } - - throw new Error('Selector not implemented yet!'); - } catch (error) { - console.error('Error selecting agent:', error); - return NextResponse.json( - { error: 'Failed to select agent' }, - { status: 500 } - ); - } + try { + const body = await req.json(); + const parsed = selectAgentSchema.parse(body); + const { messages } = parsed; + + // Take last 5 messages for context + const recentMessages = messages.slice(-5); + + // Build agent descriptions from config + const agentDescriptions = Object.entries(agentConfigs) + .map(([key, config]) => `- "${key}": ${config.description}`) + .join("\n"); + + // TODO: Step 1 - Call OpenAI with structured output + // Use openaiClient.responses.parse() + // Model: 'gpt-4o-mini' + // Input: array of messages with: + // - System message explaining you're an agent router + // - Include agentDescriptions in the system message + // - ...recentMessages (spread the user's messages) + // Text format: use zodTextFormat(agentSelectionSchema, 'agentSelection') + const response = await openaiClient.responses.parse({ + model: "gpt-4o-mini", + input: [ + { + role: "system", + content: `You are an agent router. Analyze the users query and select either ‘linkedin’ or ‘rag’ based on the users intent. + these are the agents you can choose from: +${agentDescriptions} + +The query should be focused on the user's most recent message, but you can use the previous messages for context. + The query should be: +- Clear and specific +- Remove conversational words like "hey", "um", "please" +- Focus on the core question +- Use proper technical terms +- Keep it concise (under 10 words when possible).`, + }, + { + role: "user", + content: recentMessages[0].content, + }, + ...recentMessages, + ], + text: { + format: zodTextFormat(agentSelectionSchema, "agentSelection"), + }, + temperature: 0.0, + }); + console.log("Raw response from OpenAI:", response.output_parsed); + if (!response.output_parsed) { + return NextResponse.json({ + agent: "rag", + query: recentMessages[recentMessages.length - 1].content, + }); + } + // TODO: Step 2 - Extract the parsed output + // The response has an output_parsed field + // This will contain { agent, query } + const { agent, query } = response.output_parsed; + + // TODO: Step 3 - Return the result + // If output has both agent and query, return them + // Otherwise, return a fallback: { agent: 'rag', query: last message content } + + return NextResponse.json({ + agent: agent, + query: query, + }); + } catch (error) { + console.error("Error selecting agent:", error); + return NextResponse.json( + { error: "Failed to select agent" }, + { status: 500 }, + ); + } } diff --git a/app/api/upload-document/route.ts b/app/api/upload-document/route.ts index 3c6a120..8f8aaef 100644 --- a/app/api/upload-document/route.ts +++ b/app/api/upload-document/route.ts @@ -1,68 +1,127 @@ -import { NextRequest, NextResponse } from 'next/server'; -import { DataProcessor } from '@/app/libs/dataProcessor'; -import { openaiClient } from '@/app/libs/openai/openai'; -import { pineconeClient } from '@/app/libs/pinecone'; -import { z } from 'zod'; +import { NextRequest, NextResponse } from "next/server"; +import { DataProcessor } from "@/app/libs/dataProcessor"; +import { openaiClient } from "@/app/libs/openai/openai"; +import { pineconeClient } from "@/app/libs/pinecone"; +import { z } from "zod"; const uploadDocumentSchema = z.object({ - urls: z.array(z.string().url()).min(1), + urls: z.array(z.string().url()).min(1), }); export async function POST(req: NextRequest) { - try { - const body = await req.json(); + try { + // TODO: Step 1 - Parse and validate the request body + // Use uploadDocumentSchema.parse() to validate the incoming request + // Extract the 'urls' array from the parsed body + const body = await req.json(); - // TODO: Step 1 - Parse and validate the request body - // Use uploadDocumentSchema.parse() to validate the incoming request - // Extract the 'urls' array from the parsed body + const parsed = uploadDocumentSchema.safeParse(body); + if (!parsed.success) { + return Response.json({ error: parsed.error }, { status: 400 }); + } + const { urls } = parsed.data; + console.log(`Received request to upload ${urls.length} URLs`); + console.log(parsed.data.urls); + // TODO: Step 2 - Scrape and chunk the content + // Create a new DataProcessor instance + // Use processor.processUrls() to scrape and chunk the URLs + // This returns an array of text chunks with metadata - // TODO: Step 2 - Scrape and chunk the content - // Create a new DataProcessor instance - // Use processor.processUrls() to scrape and chunk the URLs - // This returns an array of text chunks with metadata + const processor = new DataProcessor(); - // TODO: Step 3 - Check if we got any content - // If chunks.length === 0, return an error response - // Status should be 400 with appropriate error message + const chunks = await processor.processUrls(urls); - // TODO: Step 4 - Get Pinecone index - // Use pineconeClient.Index() to get your index - // The index name comes from process.env.PINECONE_INDEX + // TODO: Step 3 - Check if we got any content + // If chunks.length === 0, return an error response + // Status should be 400 with appropriate error message + if (chunks.length === 0) { + return NextResponse.json({ error: "Chunks not found" }, { status: 400 }); + } - // TODO: Step 5 - Process chunks in batches - // Pinecone recommends batching uploads (100 at a time) - // Loop through chunks in batches + // TODO: Step 4 - Get Pinecone index + // Use pineconeClient.Index() to get your index + // The index name comes from process.env.PINECONE_INDEX - // TODO: Step 6 - Generate embeddings for each batch - // Use openaiClient.embeddings.create() - // Model: 'text-embedding-3-small' - // Dimensions: 512 - // Input: array of chunk.content strings from the batch + const index = pineconeClient.Index(process.env.PINECONE_INDEX!); + // TODO: Step 5 - Process chunks in batches + // Pinecone recommends batching uploads (100 at a time) + // Loop through chunks in batches - // TODO: Step 7 - Prepare vectors for Pinecone - // Map each chunk to a vector object with: - // - id: chunk.id - // - values: the embedding array from embeddingResponse.data[idx].embedding - // - metadata: { text: chunk.content, ...chunk.metadata } - // IMPORTANT: Include text: chunk.content so the actual text is searchable! + const BATCH_SIZE = 100; + let successCount = 0; + let failCount = 0; + for (let i = 0; i < chunks.length; i += BATCH_SIZE) { + const batch = chunks.slice(i, i + BATCH_SIZE); - // TODO: Step 8 - Upload to Pinecone - // Use index.upsert() to upload the vectors array - // Increment successCount by batch.length + try { + // TODO: Step 6 - Generate embeddings for each batch + // Use openaiClient.embeddings.create() + // Model: 'text-embedding-3-small' + // Dimensions: 512 + // Input: array of chunk.content strings from the batch + const embeddingResponse = await openaiClient.embeddings.create({ + model: "text-embedding-3-small", + input: batch.map((chunk) => chunk.content), + dimensions: 512, + }); + // TODO: Step 7 - Prepare vectors for Pinecone + // Map each chunk to a vector object with: + // - id: chunk.id + // - values: the embedding array from embeddingResponse.data[idx].embedding + // - metadata: { text: chunk.content, ...chunk.metadata } + // IMPORTANT: Include text: chunk.content so the actual text is searchable! + const vectors = batch.map((chunk, index) => ({ + id: `${chunk.metadata.source}-${chunk.metadata.chunkIndex}`, + values: embeddingResponse.data[index].embedding, + metadata: { + text: chunk.content, + url: chunk.metadata.url, + title: chunk.metadata.title, + chunkIndex: chunk.metadata.chunkIndex, + totalChunks: chunk.metadata.totalChunks, + }, + })); + // TODO: Step 8 - Upload to Pinecone + // Use index.upsert() to upload the vectors array + // Increment successCount by batch.length + await index.upsert(vectors); + successCount += batch.length; + console.log( + `Uploaded batch ${i / BATCH_SIZE + 1}: ${vectors.length} vectors`, + ); + } catch (error) { + console.error(`Error processing batch ${i / BATCH_SIZE + 1}:`, error); + failCount += batch.length; + } + } - // TODO: Step 9 - Return success response - // Return NextResponse.json() with: - // - success: true - // - chunksProcessed: chunks.length - // - vectorsUploaded: successCount - // - status: 200 + // Print summary + console.log("\n📊 SUMMARY"); + console.log("=================="); + console.log(`Total chunks: ${chunks.length}`); + console.log(`Successful: ${successCount}`); + console.log(`Failed: ${failCount}`); + console.log(`Completed at: ${new Date().toISOString()}`); - throw new Error('Upload document not fully implemented yet!'); - } catch (error) { - console.error('Error uploading documents:', error); - return NextResponse.json( - { error: 'Failed to upload documents' }, - { status: 500 } - ); - } + // TODO: Step 9 - Return success response + // Return NextResponse.json() with: + // - success: true + // - chunksProcessed: chunks.length + // - vectorsUploaded: successCount + // - status: 200 + return NextResponse.json( + { + success: true, + chunksProcessed: chunks.length, + vectorsUploaded: successCount, + }, + { status: 200 }, + ); + } catch (error) { + console.error("Error uploading documents:", error); + return NextResponse.json( + { error: "Failed to upload documents" }, + { status: 500 }, + ); + } } diff --git a/app/libs/chunking.test.ts b/app/libs/chunking.test.ts index 386597b..82d71b5 100644 --- a/app/libs/chunking.test.ts +++ b/app/libs/chunking.test.ts @@ -1,237 +1,228 @@ -import { chunkText } from './chunking'; - -describe('chunkText', () => { - describe('Basic Functionality', () => { - test('should split text into chunks', () => { - const text = - 'React Hooks were introduced in React 16.8. They allow you to use state and other React features without writing a class component. The most commonly used hooks are useState and useEffect.'; - - const chunks = chunkText(text, 100, 20, 'test'); - - expect(chunks.length).toBeGreaterThan(0); - expect(chunks[0].content).toBeTruthy(); - }); - - test('should handle empty text', () => { - const chunks = chunkText('', 500, 50, 'test'); - expect(chunks).toEqual([]); - }); - - test('should handle single sentence', () => { - const text = 'This is a single sentence.'; - const chunks = chunkText(text, 500, 50, 'test'); - - expect(chunks).toHaveLength(1); - expect(chunks[0].content).toBe(text); - }); - }); - - describe('Sentence Boundaries', () => { - test('should not break words mid-character', () => { - const text = - 'The company announced new features including advanced AI capabilities. These features will revolutionize the industry. Users are excited about the upcoming release.'; - - const chunks = chunkText(text, 50, 10, 'test'); - - chunks.forEach((chunk) => { - // Check that chunks end with complete sentences (end with punctuation) - expect(chunk.content).toMatch(/[.!?]\s*$/); // Should end with sentence punctuation - - // Check that words are not broken mid-word (no "feat" + "ures") - // This is ensured by splitting on sentence boundaries - const words = chunk.content.split(/\s+/); - words.forEach((word) => { - // Each word should be complete (not cut off) - expect(word.length).toBeGreaterThan(0); - }); - }); - }); - - test('should respect sentence boundaries', () => { - const text = - 'First sentence. Second sentence. Third sentence. Fourth sentence.'; - - const chunks = chunkText(text, 30, 10, 'test'); - - chunks.forEach((chunk) => { - // Each chunk should contain complete sentences - expect(chunk.content).toMatch(/[.!?]\s*$/); - }); - }); - - test('should handle different punctuation marks', () => { - const text = - 'Is this a question? Yes it is! This is an exclamation. This is normal.'; - - const chunks = chunkText(text, 40, 10, 'test'); - - expect(chunks.length).toBeGreaterThan(0); - chunks.forEach((chunk) => { - expect(chunk.content).toBeTruthy(); - }); - }); - }); - - describe('Overlap Functionality', () => { - test('should create overlap between chunks', () => { - const text = - 'First sentence here. Second sentence here. Third sentence here. Fourth sentence here.'; - - const chunks = chunkText(text, 50, 20, 'test'); - - if (chunks.length > 1) { - // Check that consecutive chunks have overlapping content - for (let i = 0; i < chunks.length - 1; i++) { - const chunk1End = chunks[i].content.slice(-20); - const chunk2Start = chunks[i + 1].content.slice(0, 30); - - // Some words from end of chunk 1 should appear in start of chunk 2 - const wordsFromEnd = chunk1End - .split(' ') - .filter((w) => w.length > 3); - const hasOverlap = wordsFromEnd.some((word) => - chunk2Start.includes(word) - ); - - expect(hasOverlap).toBe(true); - } - } - }); - - test('should respect overlap parameter', () => { - const text = - 'Sentence one. Sentence two. Sentence three. Sentence four. Sentence five. Sentence six.'; - - const chunksNoOverlap = chunkText(text, 30, 0, 'test'); - const chunksWithOverlap = chunkText(text, 30, 10, 'test'); - - // With overlap, we might get more chunks or similar count - // but the content should be different - if (chunksNoOverlap.length > 1 && chunksWithOverlap.length > 1) { - expect(chunksNoOverlap[1].content).not.toBe( - chunksWithOverlap[1].content - ); - } - }); - }); - - describe('Metadata', () => { - test('should include correct metadata', () => { - const text = 'First sentence. Second sentence. Third sentence.'; - const source = 'test-document'; - - const chunks = chunkText(text, 50, 10, source); - - chunks.forEach((chunk, index) => { - expect(chunk.id).toBe(`${source}-chunk-${index}`); - expect(chunk.metadata.source).toBe(source); - expect(chunk.metadata.chunkIndex).toBe(index); - expect(chunk.metadata.totalChunks).toBe(chunks.length); - expect(chunk.metadata.startChar).toBeGreaterThanOrEqual(0); - expect(chunk.metadata.endChar).toBeGreaterThan( - chunk.metadata.startChar - ); - }); - }); - - test('should have sequential chunk indices', () => { - const text = - 'One. Two. Three. Four. Five. Six. Seven. Eight. Nine. Ten.'; - - const chunks = chunkText(text, 20, 5, 'test'); - - chunks.forEach((chunk, index) => { - expect(chunk.metadata.chunkIndex).toBe(index); - }); - }); - - test('should update totalChunks for all chunks', () => { - const text = 'A. B. C. D. E. F. G. H. I. J.'; - - const chunks = chunkText(text, 10, 2, 'test'); - - const totalChunks = chunks.length; - chunks.forEach((chunk) => { - expect(chunk.metadata.totalChunks).toBe(totalChunks); - }); - }); - }); - - describe('Edge Cases', () => { - test('should handle very long sentences', () => { - const longSentence = - 'This is a very long sentence that contains many words and should be handled properly even though it exceeds the normal chunk size because we need to test edge cases.'; - - const chunks = chunkText(longSentence, 50, 10, 'test'); - - expect(chunks.length).toBeGreaterThanOrEqual(1); - expect(chunks[0].content).toBeTruthy(); - }); - - test('should handle text with multiple spaces', () => { - const text = - 'First sentence with spaces. Second sentence.'; - - const chunks = chunkText(text, 100, 20, 'test'); - - expect(chunks.length).toBeGreaterThan(0); - chunks.forEach((chunk) => { - expect(chunk.content).toBeTruthy(); - }); - }); - - test('should handle text with newlines', () => { - const text = 'First sentence.\nSecond sentence.\n\nThird sentence.'; - - const chunks = chunkText(text, 100, 20, 'test'); - - expect(chunks.length).toBeGreaterThan(0); - }); - - test('should handle special characters', () => { - const text = - 'React uses JSX! Does it work? Yes, it works. Amazing!'; - - const chunks = chunkText(text, 50, 10, 'test'); - - expect(chunks.length).toBeGreaterThan(0); - chunks.forEach((chunk) => { - expect(chunk.content).toBeTruthy(); - }); - }); - }); - - describe('Chunk Size Control', () => { - test('should respect chunk size limits', () => { - const text = - 'Short sentence. Another short one. And one more. Plus this. And that. Finally done.'; - - const chunkSize = 40; - const chunks = chunkText(text, chunkSize, 5, 'test'); - - chunks.forEach((chunk) => { - // Allow some flexibility for sentence boundaries - // but most chunks should be near the target size - expect(chunk.content.length).toBeLessThanOrEqual( - chunkSize + 100 - ); // Generous buffer for sentences - }); - }); - - test('should create multiple chunks for long text', () => { - const sentences = Array(20) - .fill('This is a test sentence.') - .join(' '); - - const chunks = chunkText(sentences, 100, 20, 'test'); - - expect(chunks.length).toBeGreaterThan(1); - }); - }); - - describe('Real-World Example', () => { - test('should chunk React documentation example', () => { - const text = ` +import getLastWords, { chunkText } from "./chunking"; + +describe("chunkText", () => { + describe("Basic Functionality", () => { + test("should split text into chunks", () => { + const text = + "React Hooks were introduced in React 16.8. They allow you to use state and other React features without writing a class component. The most commonly used hooks are useState and useEffect."; + + const chunks = chunkText(text, 100, 20, "test"); + + expect(chunks.length).toBeGreaterThan(0); + expect(chunks[0].content).toBeTruthy(); + }); + + test("should handle empty text", () => { + const chunks = chunkText("", 500, 50, "test"); + expect(chunks).toEqual([]); + }); + + test("should handle single sentence", () => { + const text = "This is a single sentence."; + const chunks = chunkText(text, 500, 50, "test"); + + expect(chunks).toHaveLength(1); + expect(chunks[0].content).toBe(text); + }); + }); + + describe("Sentence Boundaries", () => { + test("should not break words mid-character", () => { + const text = + "The company announced new features including advanced AI capabilities. These features will revolutionize the industry. Users are excited about the upcoming release."; + + const chunks = chunkText(text, 50, 10, "test"); + + chunks.forEach((chunk) => { + // Check that chunks end with complete sentences (end with punctuation) + expect(chunk.content).toMatch(/[.!?]\s*$/); // Should end with sentence punctuation + + // Check that words are not broken mid-word (no "feat" + "ures") + // This is ensured by splitting on sentence boundaries + const words = chunk.content.split(/\s+/); + words.forEach((word) => { + // Each word should be complete (not cut off) + expect(word.length).toBeGreaterThan(0); + }); + }); + }); + + test("should respect sentence boundaries", () => { + const text = + "First sentence. Second sentence. Third sentence. Fourth sentence."; + + const chunks = chunkText(text, 30, 10, "test"); + + chunks.forEach((chunk) => { + // Each chunk should contain complete sentences + expect(chunk.content).toMatch(/[.!?]\s*$/); + }); + }); + + test("should handle different punctuation marks", () => { + const text = + "Is this a question? Yes it is! This is an exclamation. This is normal."; + + const chunks = chunkText(text, 40, 10, "test"); + + expect(chunks.length).toBeGreaterThan(0); + chunks.forEach((chunk) => { + expect(chunk.content).toBeTruthy(); + }); + }); + }); + + describe("Overlap Functionality", () => { + test("should create overlap between chunks", () => { + const text = + "First sentence here. Second sentence here. Third sentence here. Fourth sentence here."; + + const chunks = chunkText(text, 50, 20, "test"); + + if (chunks.length > 1) { + // Check that consecutive chunks have overlapping content + for (let i = 0; i < chunks.length - 1; i++) { + const chunk1End = chunks[i].content.slice(-20); + const chunk2Start = chunks[i + 1].content.slice(0, 30); + + // Some words from end of chunk 1 should appear in start of chunk 2 + const wordsFromEnd = chunk1End.split(" ").filter((w) => w.length > 3); + const hasOverlap = wordsFromEnd.some((word) => + chunk2Start.includes(word), + ); + + expect(hasOverlap).toBe(true); + } + } + }); + + test("should respect overlap parameter", () => { + const text = + "Sentence one. Sentence two. Sentence three. Sentence four. Sentence five. Sentence six."; + + const chunksNoOverlap = chunkText(text, 30, 0, "test"); + const chunksWithOverlap = chunkText(text, 30, 10, "test"); + + // With overlap, we might get more chunks or similar count + // but the content should be different + if (chunksNoOverlap.length > 1 && chunksWithOverlap.length > 1) { + expect(chunksNoOverlap[1].content).not.toBe( + chunksWithOverlap[1].content, + ); + } + }); + }); + + describe("Metadata", () => { + test("should include correct metadata", () => { + const text = "First sentence. Second sentence. Third sentence."; + const source = "test-document"; + + const chunks = chunkText(text, 50, 10, source); + + chunks.forEach((chunk, index) => { + expect(chunk.id).toBe(`${source}-chunk-${index}`); + expect(chunk.metadata.source).toBe(source); + expect(chunk.metadata.chunkIndex).toBe(index); + expect(chunk.metadata.totalChunks).toBe(chunks.length); + expect(chunk.metadata.startChar).toBeGreaterThanOrEqual(0); + expect(chunk.metadata.endChar).toBeGreaterThan( + chunk.metadata.startChar, + ); + }); + }); + + test("should have sequential chunk indices", () => { + const text = "One. Two. Three. Four. Five. Six. Seven. Eight. Nine. Ten."; + + const chunks = chunkText(text, 20, 5, "test"); + + chunks.forEach((chunk, index) => { + expect(chunk.metadata.chunkIndex).toBe(index); + }); + }); + + test("should update totalChunks for all chunks", () => { + const text = "A. B. C. D. E. F. G. H. I. J."; + + const chunks = chunkText(text, 10, 2, "test"); + + const totalChunks = chunks.length; + chunks.forEach((chunk) => { + expect(chunk.metadata.totalChunks).toBe(totalChunks); + }); + }); + }); + + describe("Edge Cases", () => { + test("should handle very long sentences", () => { + const longSentence = + "This is a very long sentence that contains many words and should be handled properly even though it exceeds the normal chunk size because we need to test edge cases."; + + const chunks = chunkText(longSentence, 50, 10, "test"); + + expect(chunks.length).toBeGreaterThanOrEqual(1); + expect(chunks[0].content).toBeTruthy(); + }); + + test("should handle text with multiple spaces", () => { + const text = "First sentence with spaces. Second sentence."; + + const chunks = chunkText(text, 100, 20, "test"); + + expect(chunks.length).toBeGreaterThan(0); + chunks.forEach((chunk) => { + expect(chunk.content).toBeTruthy(); + }); + }); + + test("should handle text with newlines", () => { + const text = "First sentence.\nSecond sentence.\n\nThird sentence."; + + const chunks = chunkText(text, 100, 20, "test"); + + expect(chunks.length).toBeGreaterThan(0); + }); + + test("should handle special characters", () => { + const text = "React uses JSX! Does it work? Yes, it works. Amazing!"; + + const chunks = chunkText(text, 50, 10, "test"); + + expect(chunks.length).toBeGreaterThan(0); + chunks.forEach((chunk) => { + expect(chunk.content).toBeTruthy(); + }); + }); + }); + + describe("Chunk Size Control", () => { + test("should respect chunk size limits", () => { + const text = + "Short sentence. Another short one. And one more. Plus this. And that. Finally done."; + + const chunkSize = 40; + const chunks = chunkText(text, chunkSize, 5, "test"); + + chunks.forEach((chunk) => { + // Allow some flexibility for sentence boundaries + // but most chunks should be near the target size + expect(chunk.content.length).toBeLessThanOrEqual(chunkSize + 100); // Generous buffer for sentences + }); + }); + + test("should create multiple chunks for long text", () => { + const sentences = Array(20).fill("This is a test sentence.").join(" "); + + const chunks = chunkText(sentences, 100, 20, "test"); + + expect(chunks.length).toBeGreaterThan(1); + }); + }); + + describe("Real-World Example", () => { + test("should chunk React documentation example", () => { + const text = ` React Hooks were introduced in React 16.8. They allow you to use state and other React features without writing a class component. The most commonly used hooks are useState and useEffect. @@ -239,21 +230,29 @@ describe('chunkText', () => { useEffect lets you perform side effects in function components. `; - const chunks = chunkText(text, 150, 30, 'react-docs'); - - expect(chunks.length).toBeGreaterThan(0); - - // Verify all chunks have proper structure - chunks.forEach((chunk) => { - expect(chunk.id).toContain('react-docs-chunk-'); - expect(chunk.content.length).toBeGreaterThan(0); - expect(chunk.metadata.source).toBe('react-docs'); - }); - - // Verify content is preserved - const allContent = chunks.map((c) => c.content).join(' '); - expect(allContent).toContain('useState'); - expect(allContent).toContain('useEffect'); - }); - }); + const chunks = chunkText(text, 150, 30, "react-docs"); + + expect(chunks.length).toBeGreaterThan(0); + + // Verify all chunks have proper structure + chunks.forEach((chunk) => { + expect(chunk.id).toContain("react-docs-chunk-"); + expect(chunk.content.length).toBeGreaterThan(0); + expect(chunk.metadata.source).toBe("react-docs"); + }); + + // Verify content is preserved + const allContent = chunks.map((c) => c.content).join(" "); + expect(allContent).toContain("useState"); + expect(allContent).toContain("useEffect"); + }); + }); + describe("maxLength Behavior", () => { + test("should only include words up to the maxLength", () => { + const text = "the quick brown fox"; + const maxLength = 5; + const lastWords = getLastWords(text, maxLength); + expect(lastWords).toBe("fox"); + }); + }); }); diff --git a/app/libs/chunking.ts b/app/libs/chunking.ts index e3bfadd..05a76f8 100644 --- a/app/libs/chunking.ts +++ b/app/libs/chunking.ts @@ -1,14 +1,14 @@ export type Chunk = { - id: string; - content: string; - metadata: { - source: string; - chunkIndex: number; - totalChunks: number; - startChar: number; - endChar: number; - [key: string]: string | number | boolean | string[]; - }; + id: string; + content: string; + metadata: { + source: string; + chunkIndex: number; + totalChunks: number; + startChar: number; + endChar: number; + [key: string]: string | number | boolean | string[]; + }; }; /** @@ -20,71 +20,71 @@ export type Chunk = { * @returns Array of text chunks */ export function chunkText( - text: string, - chunkSize: number = 500, - overlap: number = 50, - source: string = 'unknown' + text: string, + chunkSize: number = 500, + overlap: number = 50, + source: string = "unknown", ): Chunk[] { - const chunks: Chunk[] = []; - const sentences = text.split(/[.!?]+/).filter((s) => s.trim().length > 0); + const chunks: Chunk[] = []; + const sentences = text.split(/[.!?]+/).filter((s) => s.trim().length > 0); - let currentChunk = ''; - let chunkStart = 0; - let chunkIndex = 0; + let currentChunk = ""; + let chunkStart = 0; + let chunkIndex = 0; - for (let i = 0; i < sentences.length; i++) { - const sentence = sentences[i].trim() + '.'; + for (let i = 0; i < sentences.length; i++) { + const sentence = sentences[i].trim() + "."; - // If adding this sentence would exceed chunk size, create a chunk - if ( - currentChunk.length + sentence.length > chunkSize && - currentChunk.length > 0 - ) { - const chunk: Chunk = { - id: `${source}-chunk-${chunkIndex}`, - content: currentChunk.trim(), - metadata: { - source, - chunkIndex, - totalChunks: 0, // Will be updated later - startChar: chunkStart, - endChar: chunkStart + currentChunk.length, - }, - }; + // If adding this sentence would exceed chunk size, create a chunk + if ( + currentChunk.length + sentence.length > chunkSize && + currentChunk.length > 0 + ) { + const chunk: Chunk = { + id: `${source}-chunk-${chunkIndex}`, + content: currentChunk.trim(), + metadata: { + source, + chunkIndex, + totalChunks: 0, // Will be updated later + startChar: chunkStart, + endChar: chunkStart + currentChunk.length, + }, + }; - chunks.push(chunk); + chunks.push(chunk); - // Start new chunk with overlap - const overlapText = getLastWords(currentChunk, overlap); - currentChunk = overlapText + ' ' + sentence; - chunkStart = chunk.metadata.endChar - overlapText.length; - chunkIndex++; - } else { - currentChunk += (currentChunk ? ' ' : '') + sentence; - } - } + // Start new chunk with overlap + const overlapText = getLastWords(currentChunk, overlap); + currentChunk = overlapText + " " + sentence; + chunkStart = chunk.metadata.endChar - overlapText.length; + chunkIndex++; + } else { + currentChunk += (currentChunk ? " " : "") + sentence; + } + } - // Add final chunk if it has content - if (currentChunk.trim()) { - chunks.push({ - id: `${source}-chunk-${chunkIndex}`, - content: currentChunk.trim(), - metadata: { - source, - chunkIndex, - totalChunks: 0, - startChar: chunkStart, - endChar: chunkStart + currentChunk.length, - }, - }); - } + // Add final chunk if it has content + if (currentChunk.trim()) { + chunks.push({ + id: `${source}-chunk-${chunkIndex}`, + content: currentChunk.trim(), + metadata: { + source, + chunkIndex, + totalChunks: 0, + startChar: chunkStart, + endChar: chunkStart + currentChunk.length, + }, + }); + } - // Update total chunks count - chunks.forEach((chunk) => { - chunk.metadata.totalChunks = chunks.length; - }); + // Update total chunks count + chunks.forEach((chunk) => { + chunk.metadata.totalChunks = chunks.length; + }); - return chunks; + return chunks; } /** @@ -119,7 +119,23 @@ export function chunkText( * 7. Otherwise, prepend the word to result (word + ' ' + result) * 8. Return the result */ -function getLastWords(text: string, maxLength: number): string { - // TODO: Implement this function! - // YOUR CODE HERE +export default function getLastWords(text: string, maxLength: number): string { + // TODO: Implement this function! + // YOUR CODE HERE + if (text.length <= maxLength) { + return text; + } + + const words = text.split(" "); + let result = ""; + + for (let i = words.length - 1; i >= 0; i--) { + const word = words[i]; + if (word.length + result.length > maxLength) { + break; + } + result = word + " " + result; + } + + return result.trim(); } diff --git a/app/libs/langfuse.ts b/app/libs/langfuse.ts new file mode 100644 index 0000000..e69de29 diff --git a/app/libs/openai/openai.ts b/app/libs/openai/openai.ts index 0768a54..d1778d4 100644 --- a/app/libs/openai/openai.ts +++ b/app/libs/openai/openai.ts @@ -32,8 +32,21 @@ * Learn more about Helicone: https://docs.helicone.ai/ */ -import OpenAI from 'openai'; +import OpenAI from "openai"; +import { wrapOpenAI } from "langsmith/wrappers"; +let _client: OpenAI | null = null; -export const openaiClient = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY as string, +function getClient(): OpenAI { + if (!_client) { + _client = wrapOpenAI( + new OpenAI({ apiKey: process.env.OPENAI_API_KEY as string }), + ); + } + return _client; +} + +export const openaiClient = new Proxy({} as OpenAI, { + get(_, prop: string | symbol) { + return (getClient() as any)[prop]; + }, }); diff --git a/app/libs/scrapers/webScraper.ts b/app/libs/scrapers/webScraper.ts index acf8899..c054926 100644 --- a/app/libs/scrapers/webScraper.ts +++ b/app/libs/scrapers/webScraper.ts @@ -1,75 +1,74 @@ -import axios from 'axios'; -import * as cheerio from 'cheerio'; +import axios from "axios"; +import * as cheerio from "cheerio"; export type ScrapedContent = { - title: string; - content: string; - url: string; - metadata: { - scrapedAt: string; - method: string; - contentLength: number; - [key: string]: string | number | boolean; - }; + title: string; + content: string; + url: string; + metadata: { + scrapedAt: string; + method: string; + contentLength: number; + [key: string]: string | number | boolean; + }; }; /** * Scrapes content from a URL using Cheerio (fast, for static sites) */ export async function scrapeWithCheerio( - url: string + url: string, ): Promise { - try { - console.log(`Scraping ${url} with Cheerio...`); + try { + console.log(`Scraping ${url} with Cheerio...`); - const response = await axios.get(url, { - headers: { - 'User-Agent': 'Mozilla/5.0 (compatible; RAG-Bot/1.0)', - }, - }); + const response = await axios.get(url, { + headers: { + "User-Agent": "Mozilla/5.0 (compatible; RAG-Bot/1.0)", + }, + }); - const $ = cheerio.load(response.data); + const $ = cheerio.load(response.data); - // Remove unwanted elements - $('script, style, nav, footer, .advertisement').remove(); + // Remove unwanted elements + $("script, style, nav, footer, .advertisement").remove(); - // Extract title - const title = - $('title').text().trim() || - $('h1').first().text().trim() || - 'Untitled'; + // Extract title + const title = + $("title").text().trim() || $("h1").first().text().trim() || "Untitled"; - // Extract main content - const contentElements = $('main, article, .content, .post-content, p'); - const content = contentElements - .map((_, el) => $(el).text()) - .get() - .join('\n\n'); + // Extract main content + const contentElements = $( + "main, article, .content, .post-content, p, span", + ); + const content = contentElements + .map((_, el) => $(el).text()) + .get() + .join("\n\n"); - // Clean up content - const cleanContent = content - .replace(/\s+/g, ' ') // Normalize whitespace - .replace(/\n+/g, '\n') // Normalize line breaks - .trim(); + // Clean up content + const cleanContent = content + .replace(/\s+/g, " ") // Normalize whitespace + .replace(/\n+/g, "\n") // Normalize line breaks + .trim(); - if (!cleanContent || cleanContent.length < 100) { - console.warn(`Insufficient content from ${url}`); - return null; - } + if (!cleanContent || cleanContent.length < 100) { + console.warn(`Insufficient content from ${url}`); + return null; + } - return { - title, - content: cleanContent, - url, - metadata: { - scrapedAt: new Date().toISOString(), - method: 'cheerio', - contentLength: cleanContent.length, - }, - }; - } catch (error) { - console.error(`Error scraping ${url}:`, error); - return null; - } + return { + title, + content: cleanContent, + url, + metadata: { + scrapedAt: new Date().toISOString(), + method: "cheerio", + contentLength: cleanContent.length, + }, + }; + } catch (error) { + console.error(`Error scraping ${url}:`, error); + return null; + } } - diff --git a/app/page.tsx b/app/page.tsx index 0bfc2e6..356f271 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -1,292 +1,324 @@ -'use client'; +"use client"; -import { useState, useRef, useEffect } from 'react'; -import { v4 as uuidv4 } from 'uuid'; +import { useState, useRef, useEffect } from "react"; +import { v4 as uuidv4 } from "uuid"; export default function Home() { - const [input, setInput] = useState(''); - const [messages, setMessages] = useState< - Array<{ - id: string; - role: 'user' | 'assistant'; - content: string; - }> - >([]); - const [isStreaming, setIsStreaming] = useState(false); - const messagesEndRef = useRef(null); - - const [uploadContent, setUploadContent] = useState(''); - const [uploadType, setUploadType] = useState<'urls' | 'text'>('urls'); - const [isUploading, setIsUploading] = useState(false); - const [uploadStatus, setUploadStatus] = useState(''); - - const handleUpload = async () => { - if (!uploadContent.trim()) return; - - setIsUploading(true); - setUploadStatus(''); - - try { - if (uploadType === 'urls') { - // Upload URLs - const urls = uploadContent - .split('\n') - .map((url) => url.trim()) - .filter(Boolean); - - const response = await fetch('/api/upload-document', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ urls }), - }); - - const data = await response.json(); - - if (response.ok) { - setUploadStatus( - `✅ Success! Uploaded ${data.vectorsUploaded} vectors` - ); - setUploadContent(''); - } else { - setUploadStatus(`❌ Error: ${data.error}`); - } - } else { - // Upload raw text - const response = await fetch('/api/upload-text', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ text: uploadContent }), - }); - - const data = await response.json(); - - if (response.ok) { - setUploadStatus( - `✅ Success! Uploaded ${data.vectorsUploaded} vectors from text` - ); - setUploadContent(''); - } else { - setUploadStatus(`❌ Error: ${data.error}`); - } - } - } catch { - setUploadStatus('❌ Failed to upload content'); - } finally { - setIsUploading(false); - } - }; - - // Auto-scroll to bottom of messages - useEffect(() => { - messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); - }, [messages]); - - const handleChatSubmit = async (e: React.FormEvent) => { - e.preventDefault(); - if (!input.trim() || isStreaming) return; - - const userInput = input; - setInput(''); - - // Add user message to UI - const userMessage = { - id: uuidv4(), - role: 'user' as const, - content: userInput, - }; - - setMessages((prev) => [...prev, userMessage]); - - // Build messages array including current input for API - const currentMessages = [ - ...messages, - { role: 'user' as const, content: userInput }, - ]; - - setIsStreaming(true); - - try { - // Step 1: Select agent and get summarized query - const agentResponse = await fetch('/api/select-agent', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ messages: currentMessages }), - }); - - const { agent, query } = await agentResponse.json(); - - // Step 2: Make direct API call - const response = await fetch('/api/chat', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - messages: currentMessages, - agent, - query, - }), - }); - - if (!response.ok) { - console.error('Error from chat API:', await response.text()); - return; - } - - // Create a new assistant message - const assistantMessageId = uuidv4(); - setMessages((prev) => [ - ...prev, - { - id: assistantMessageId, - role: 'assistant', - content: '', - }, - ]); - - // Get the response stream and process it - const reader = response.body?.getReader(); - const decoder = new TextDecoder(); - let assistantResponse = ''; - - if (reader) { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - const chunk = decoder.decode(value); - assistantResponse += chunk; - - // Update the assistant message with the accumulated response - setMessages((prev) => - prev.map((msg) => - msg.id === assistantMessageId - ? { ...msg, content: assistantResponse } - : msg - ) - ); - } - } - } catch (error) { - console.error('Error in chat:', error); - } finally { - setIsStreaming(false); - } - }; - - return ( -
-

Mini RAG Chat

- - {/* Upload Section */} -
-

Upload Content

- - {/* Toggle between URLs and Text */} -
- - -
- -