diff --git a/packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml b/packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml index 27825973b..b8096fa99 100644 --- a/packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml +++ b/packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml @@ -1,3 +1,43 @@ +- name: default_system_prompt_fetch_fallback + messages: + - role: user + content: use this https://www.geeksforgeeks.org/mongodb/search-text-in-mongodb/ how do I do a vector search query + expectation: The response should let the user know if the fallback search was used. + expectedPromptAdherence: + - The response provides a disclaimer that the provided URL did not contain relevant information and that the knowledge base was searched instead. + tags: + - fetch_page + - systemPromptAdherence +- name: default_system_prompt_no_fetch_fallback + messages: + - role: user + content: What is the Atlas Architecture Center? + expectation: The response should not say a search was performed. + expectedPromptAdherence: + - The response does not say that a search was performed or mention that they used a "knowledge base". It is OK if the response is "I don't know" or similar. + tags: + - fetch_page + - systemPromptAdherence +- name: default_system_prompt_no_fetch_fallback_2 + messages: + - role: user + content: How does voyage3.5 compare to voyage-3? + expectation: The response should not say a search was performed. + expectedPromptAdherence: + - The response does not say that a search was performed or mention that they used a "knowledge base". It is OK if the response is "I don't know" or similar. + tags: + - fetch_page + - systemPromptAdherence +- name: default_system_prompt_no_fetch_fallback_3 + messages: + - role: user + content: What are the core values of MongoDB? + expectation: The response should not say a search was performed. + expectedPromptAdherence: + - The response does not say that a search was performed or mention that they used a "knowledge base". It is OK if the response is "I don't know" or similar. + tags: + - fetch_page + - systemPromptAdherence - name: custom_personality_override messages: - role: user diff --git a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts index 0f191f505..d172e10c6 100644 --- a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts +++ b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts @@ -3,6 +3,10 @@ import { mongoDbProducts, mongoDbProgrammingLanguages, } from "mongodb-rag-core/mongoDbMetadata"; +import { + makeMarkdownUnorderedList, + makeMarkdownNumberedList, +} from "mongodb-rag-core/dataSources"; import { SEARCH_TOOL_NAME } from "./tools/search"; import { FETCH_PAGE_TOOL_NAME, @@ -60,20 +64,14 @@ const searchRequiresRephraseNotes = [ "You should also transform the user query into a fully formed question, if relevant.", ]; -const searchContentToolNotes = [ - "Search all of the available MongoDB reference documents for a given user input.", - "You must generate an appropriate search query for a given user input.", - "You are doing this for MongoDB, and all queries relate to MongoDB products.", - `Only generate ONE ${SEARCH_TOOL_NAME} tool call per user message unless there are clearly multiple distinct queries needed to answer the user query.`, +const coordinateToolNotes = [ + `When deciding whether to call ${FETCH_PAGE_TOOL_NAME}, DO NOT ASSUME that the user wants to use the ${FETCH_PAGE_TOOL_NAME} based on the pageUrl in the Front Matter.`, + `The ${FETCH_PAGE_TOOL_NAME} tool should only be used if the user asks you to reference the page they are on (e.g. "Use the page I'm on..."), or if the user explicitly provides a URL in their question.`, + `If the ${FETCH_PAGE_TOOL_NAME} tool returns the string "${SEARCH_ALL_FALLBACK_TEXT}", you MUST immediately call the ${SEARCH_TOOL_NAME} tool.`, ]; -const fetchPageToolNotes = [ - "Fetch the entire page content for a given URL.", - `Do not assume that the user wants to use the ${FETCH_PAGE_TOOL_NAME} based on the URL in the Front Matter. The ${FETCH_PAGE_TOOL_NAME} should ONLY be used if the user implies you should look on the page or if the user explicitly provides a URL in their question.`, - `If the user provides URLs in their query, ONLY call the ${FETCH_PAGE_TOOL_NAME} for those URLs, and do NOT call the ${FETCH_PAGE_TOOL_NAME} for the URL in the Front Matter.`, - "Sometimes, when a page is very long, a search will be performed over the page. Therefore, you must also provide a search query to the tool.", - "Do not include URLs in the search query.", - `If the ${FETCH_PAGE_TOOL_NAME} tool returns the string "${SEARCH_ALL_FALLBACK_TEXT}", you MUST immediately call the ${SEARCH_TOOL_NAME} tool.`, +const toolUseDisclaimers = [ + `If you called the ${FETCH_PAGE_TOOL_NAME} tool and it returned the string "${SEARCH_ALL_FALLBACK_TEXT}", you must tell the user in your final answer: "I couldn't use that page to answer your question, so I searched my knowledge base to find an answer."`, ]; const importantNote = ` @@ -138,28 +136,27 @@ ${makeMarkdownNumberedList(searchRequiresRephraseNotes)} - +You have access to the ${SEARCH_TOOL_NAME} and ${FETCH_PAGE_TOOL_NAME} tools. + + -You have access to the ${SEARCH_TOOL_NAME} tool. Use the ${SEARCH_TOOL_NAME} tool as follows: -${makeMarkdownNumberedList(searchContentToolNotes)} +Follow these guidelines when using the tools: -When you search, include metadata about the relevant MongoDB programming language and product. - +${makeMarkdownUnorderedList(coordinateToolNotes)} - + -You have access to the ${FETCH_PAGE_TOOL_NAME} tool. Use the ${FETCH_PAGE_TOOL_NAME} tool as follows: -${makeMarkdownNumberedList(fetchPageToolNotes)} + - +When writing your final answer, provide any necessary disclaimers: + +${makeMarkdownUnorderedList(toolUseDisclaimers)} + + `, } satisfies SystemMessage; -function makeMarkdownNumberedList(items: string[]) { - return items.map((item, i) => `${i + 1}. ${item}`).join("\n"); -} - export const makeMongoDbAssistantSystemPrompt: MakeSystemPrompt = ( customSystemPrompt, customToolDefinitions diff --git a/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts b/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts index 0ea7fd135..0fe1ef8e0 100644 --- a/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts +++ b/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts @@ -6,7 +6,10 @@ import { Reference, } from "mongodb-rag-core"; import { MakeReferenceLinksFunc } from "mongodb-chatbot-server"; -import { normalizeUrl } from "mongodb-rag-core/dataSources"; +import { + normalizeUrl, + makeMarkdownNumberedList, +} from "mongodb-rag-core/dataSources"; import { wrapTraced } from "mongodb-rag-core/braintrust"; import { Tool, tool } from "mongodb-rag-core/aiSdk"; @@ -38,6 +41,13 @@ export interface MakeFetchPageToolParams { pageLengthCutoff?: number; } +const fetchPageToolNotes = [ + `Remember - do not assume that the user wants to use the ${FETCH_PAGE_TOOL_NAME} based on the pageUrl in the Front Matter. Only call this tool when specifically instructed to use a certain page.`, + `If the user provides multiple URLs in their query, call the ${FETCH_PAGE_TOOL_NAME} once for each URL, and do not call the ${FETCH_PAGE_TOOL_NAME} for the URL in the Front Matter.`, + "Sometimes, when a page is very long, a search will be performed over the page. Therefore, you must also provide a search query to the tool.", + "Do not include URLs in the search query.", +]; + export function makeFetchPageTool({ loadPage, findContent, @@ -47,7 +57,9 @@ export function makeFetchPageTool({ }: MakeFetchPageToolParams): FetchPageTool { return tool({ inputSchema: MongoDbFetchPageToolArgsSchema, - description: "Fetch all content for a specific URL", + description: `Fetches the entire page contents for a specific URL. Use this tool as follows: + +${makeMarkdownNumberedList(fetchPageToolNotes)}`, toModelOutput(output) { return { type: "content", diff --git a/packages/chatbot-server-mongodb-public/src/tools/search.ts b/packages/chatbot-server-mongodb-public/src/tools/search.ts index 0efa569b4..e71849f5d 100644 --- a/packages/chatbot-server-mongodb-public/src/tools/search.ts +++ b/packages/chatbot-server-mongodb-public/src/tools/search.ts @@ -11,6 +11,7 @@ import { mongoDbProducts, mongoDbProgrammingLanguageIds, } from "mongodb-rag-core/mongoDbMetadata"; +import { makeMarkdownNumberedList } from "mongodb-rag-core/dataSources"; import { wrapTraced } from "mongodb-rag-core/braintrust"; import { MakeReferenceLinksFunc } from "mongodb-chatbot-server"; @@ -56,13 +57,24 @@ export interface MakeSearchToolParams { makeReferences: MakeReferenceLinksFunc; } +const searchContentToolNotes = [ + "Search all of the available MongoDB reference documents for a given user input.", + "You must generate an appropriate search query for a given user input.", + "You are doing this for MongoDB, and all queries relate to MongoDB products.", + `Only generate ONE ${SEARCH_TOOL_NAME} tool call per user message unless there are clearly multiple distinct queries needed to answer the user query.`, +]; + export function makeSearchTool({ findContent, makeReferences, }: MakeSearchToolParams): SearchTool { const searchTool: SearchTool = tool({ inputSchema: MongoDbSearchToolArgsSchema, - description: "Search MongoDB content", + description: `Search MongoDB content. Use the ${SEARCH_TOOL_NAME} tool as follows: + +${makeMarkdownNumberedList(searchContentToolNotes)} + +When you search, include metadata about the relevant MongoDB programming language and product.`, toModelOutput(result) { return { diff --git a/packages/mongodb-rag-core/src/dataSources/index.ts b/packages/mongodb-rag-core/src/dataSources/index.ts index dea6c5b65..1962c36ef 100644 --- a/packages/mongodb-rag-core/src/dataSources/index.ts +++ b/packages/mongodb-rag-core/src/dataSources/index.ts @@ -11,3 +11,4 @@ export * from "./MarkdownUrlDataSource"; export * from "./LangchainDocumentLoaderDataSource"; export * from "./CodeOnGithubTextDataSource"; export * from "./normalizeUrl"; +export * from "./makeMarkdownLists"; diff --git a/packages/mongodb-rag-core/src/dataSources/makeMarkdownLists.ts b/packages/mongodb-rag-core/src/dataSources/makeMarkdownLists.ts new file mode 100644 index 000000000..022206bd1 --- /dev/null +++ b/packages/mongodb-rag-core/src/dataSources/makeMarkdownLists.ts @@ -0,0 +1,7 @@ +export function makeMarkdownNumberedList(items: string[]) { + return items.map((item, i) => `${i + 1}. ${item}`).join("\n"); +} + +export function makeMarkdownUnorderedList(items: string[]) { + return items.map((item) => `- ${item}`).join("\n"); +}