From a5a89b9fa1420adf4abe6629ca41678ac30f25f4 Mon Sep 17 00:00:00 2001 From: Helen Schawe Date: Fri, 22 Aug 2025 09:05:38 -0500 Subject: [PATCH 01/12] fetch_page fallback disclaimer instruction --- packages/chatbot-server-mongodb-public/src/systemPrompt.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts index 0f191f505..c725cf770 100644 --- a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts +++ b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts @@ -18,6 +18,9 @@ export type MakeSystemPrompt = ( export const llmDoesNotKnowMessage = "I'm sorry, I do not know how to answer that question. Please try to rephrase your query."; +const fallbackDisclaimer = + "I didn't find anything on the page you provided, so I did a search of all MongoDB content instead."; + const chatbotOverview = `You are expert MongoDB documentation chatbot.`; const personalityTraits = [ @@ -44,6 +47,7 @@ const technicalKnowledge = [ const importantNotes = [ `ALWAYS use either ${SEARCH_TOOL_NAME} or ${FETCH_PAGE_TOOL_NAME} after every user message. Zero exceptions!`, + `If you used the ${FETCH_PAGE_TOOL_NAME} tool and had to fall back to ${SEARCH_TOOL_NAME}, tell the user you didn't find anything on the page they provided, and that you searched all MongoDB content instead.`, ]; const metadataNotes = [ @@ -94,6 +98,9 @@ ${makeMarkdownNumberedList(personalityTraits)} If you do not know the answer to the question, respond only with the following text: "${llmDoesNotKnowMessage}" +When crafting your final answer to the user, follow these guidelines: +${importantNote} + Response format: ${makeMarkdownNumberedList(responseFormat)} From d65ddcb747d08ab60d96f57ab8b2e55fdbcc84a3 Mon Sep 17 00:00:00 2001 From: Helen Schawe Date: Fri, 22 Aug 2025 14:07:35 -0500 Subject: [PATCH 02/12] Remove unused str & tweaks to make prompt encapsulate successful fetch_page calls --- packages/chatbot-server-mongodb-public/src/systemPrompt.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts index c725cf770..89d6ac0c3 100644 --- a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts +++ b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts @@ -18,9 +18,6 @@ export type MakeSystemPrompt = ( export const llmDoesNotKnowMessage = "I'm sorry, I do not know how to answer that question. Please try to rephrase your query."; -const fallbackDisclaimer = - "I didn't find anything on the page you provided, so I did a search of all MongoDB content instead."; - const chatbotOverview = `You are expert MongoDB documentation chatbot.`; const personalityTraits = [ @@ -47,7 +44,7 @@ const technicalKnowledge = [ const importantNotes = [ `ALWAYS use either ${SEARCH_TOOL_NAME} or ${FETCH_PAGE_TOOL_NAME} after every user message. Zero exceptions!`, - `If you used the ${FETCH_PAGE_TOOL_NAME} tool and had to fall back to ${SEARCH_TOOL_NAME}, tell the user you didn't find anything on the page they provided, and that you searched all MongoDB content instead.`, + `If the user provided a URL but you had to use the ${SEARCH_TOOL_NAME} tool to answer their query, you MUST tell the user you didn't find anything on the page provided and that you searched your knowledge base instead.`, ]; const metadataNotes = [ From 20d1717e4fb82a8cba55ead82c192ba3c62d0f76 Mon Sep 17 00:00:00 2001 From: Helen Schawe Date: Mon, 25 Aug 2025 15:16:48 -0500 Subject: [PATCH 03/12] PR review - importantNote shouldn't be in the systemPromptContent --- packages/chatbot-server-mongodb-public/src/systemPrompt.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts index 89d6ac0c3..1f10ccd29 100644 --- a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts +++ b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts @@ -44,7 +44,6 @@ const technicalKnowledge = [ const importantNotes = [ `ALWAYS use either ${SEARCH_TOOL_NAME} or ${FETCH_PAGE_TOOL_NAME} after every user message. Zero exceptions!`, - `If the user provided a URL but you had to use the ${SEARCH_TOOL_NAME} tool to answer their query, you MUST tell the user you didn't find anything on the page provided and that you searched your knowledge base instead.`, ]; const metadataNotes = [ @@ -77,6 +76,10 @@ const fetchPageToolNotes = [ `If the ${FETCH_PAGE_TOOL_NAME} tool returns the string "${SEARCH_ALL_FALLBACK_TEXT}", you MUST immediately call the ${SEARCH_TOOL_NAME} tool.`, ]; +const finalAnswerContentNotes = [ + `If the user provided a URL but you had to use the ${SEARCH_TOOL_NAME} tool to answer their query, you MUST tell the user you didn't find anything on the page provided and that you searched your knowledge base instead.` +]; + const importantNote = ` ${makeMarkdownNumberedList(importantNotes)} `; @@ -96,7 +99,7 @@ If you do not know the answer to the question, respond only with the following t "${llmDoesNotKnowMessage}" When crafting your final answer to the user, follow these guidelines: -${importantNote} +${makeMarkdownNumberedList(finalAnswerContentNotes)} Response format: ${makeMarkdownNumberedList(responseFormat)} From b675ff520ecff49bce1ccb1e5c9cfde747362d29 Mon Sep 17 00:00:00 2001 From: Helen Schawe Date: Tue, 26 Aug 2025 10:08:40 -0500 Subject: [PATCH 04/12] Add 1 prompt adherence eval case for fallback instruction --- .../evalCases/system_prompt_conversations.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml b/packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml index 27825973b..895ad3b62 100644 --- a/packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml +++ b/packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml @@ -1,3 +1,13 @@ +- name: default_system_prompt_fetch_fallback + messages: + - role: user + content: use this https://www.geeksforgeeks.org/mongodb/search-text-in-mongodb/ how do I do a vector search query + expectation: The response should adhere to the default system prompt (let the user know if the fallback search was used). + expectedPromptAdherence: + - The response provides a disclaimer that the provided URL did not contain relevant information and that the knowledge base was searched instead. + tags: + - fetch_page + - systemPromptAdherence - name: custom_personality_override messages: - role: user From 06ed18c8132aed904d4b504f1feb950cbf9dad61 Mon Sep 17 00:00:00 2001 From: Helen Schawe Date: Tue, 26 Aug 2025 15:03:34 -0500 Subject: [PATCH 05/12] Test run: Tool instructions in tool descriptions --- .../evalCases/system_prompt_conversations.yml | 32 ++++++++++++++++++- .../src/conversations.eval.ts | 4 +-- .../src/systemPrompt.ts | 20 ++---------- .../src/tools/fetchPage.ts | 19 ++++++++++- .../src/tools/search.ts | 17 +++++++++- 5 files changed, 70 insertions(+), 22 deletions(-) diff --git a/packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml b/packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml index 895ad3b62..b8096fa99 100644 --- a/packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml +++ b/packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml @@ -2,12 +2,42 @@ messages: - role: user content: use this https://www.geeksforgeeks.org/mongodb/search-text-in-mongodb/ how do I do a vector search query - expectation: The response should adhere to the default system prompt (let the user know if the fallback search was used). + expectation: The response should let the user know if the fallback search was used. expectedPromptAdherence: - The response provides a disclaimer that the provided URL did not contain relevant information and that the knowledge base was searched instead. tags: - fetch_page - systemPromptAdherence +- name: default_system_prompt_no_fetch_fallback + messages: + - role: user + content: What is the Atlas Architecture Center? + expectation: The response should not say a search was performed. + expectedPromptAdherence: + - The response does not say that a search was performed or mention that they used a "knowledge base". It is OK if the response is "I don't know" or similar. + tags: + - fetch_page + - systemPromptAdherence +- name: default_system_prompt_no_fetch_fallback_2 + messages: + - role: user + content: How does voyage3.5 compare to voyage-3? + expectation: The response should not say a search was performed. + expectedPromptAdherence: + - The response does not say that a search was performed or mention that they used a "knowledge base". It is OK if the response is "I don't know" or similar. + tags: + - fetch_page + - systemPromptAdherence +- name: default_system_prompt_no_fetch_fallback_3 + messages: + - role: user + content: What are the core values of MongoDB? + expectation: The response should not say a search was performed. + expectedPromptAdherence: + - The response does not say that a search was performed or mention that they used a "knowledge base". It is OK if the response is "I don't know" or similar. + tags: + - fetch_page + - systemPromptAdherence - name: custom_personality_override messages: - role: user diff --git a/packages/chatbot-server-mongodb-public/src/conversations.eval.ts b/packages/chatbot-server-mongodb-public/src/conversations.eval.ts index 48ab6c381..695cbc287 100644 --- a/packages/chatbot-server-mongodb-public/src/conversations.eval.ts +++ b/packages/chatbot-server-mongodb-public/src/conversations.eval.ts @@ -64,13 +64,13 @@ async function conversationEval() { ...systemPromptCases, ...internalToolCases, ...customToolCases, - ]; + ].filter((c) => c.tags?.includes("fetch_page") || c.tags?.includes("systemPromptAdherence")); try { // Run the conversation eval const evalResult = await makeConversationEval({ projectName: CONVERSATION_EVAL_PROJECT_NAME, - experimentName: "mongodb-chatbot-latest", + experimentName: "mongodb-chatbot-feature-fetch-adherence", metadata: { description: "Evaluates how well the MongoDB AI Chatbot RAG pipeline works", diff --git a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts index 1f10ccd29..2fd5ebdee 100644 --- a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts +++ b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts @@ -77,7 +77,7 @@ const fetchPageToolNotes = [ ]; const finalAnswerContentNotes = [ - `If the user provided a URL but you had to use the ${SEARCH_TOOL_NAME} tool to answer their query, you MUST tell the user you didn't find anything on the page provided and that you searched your knowledge base instead.` + `If the ${FETCH_PAGE_TOOL_NAME} tool replies with "${SEARCH_ALL_FALLBACK_TEXT}", you should tell the user you didn't find anything on the page provided and that you searched your knowledge base to find an answer.`, ]; const importantNote = ` @@ -98,9 +98,6 @@ ${makeMarkdownNumberedList(personalityTraits)} If you do not know the answer to the question, respond only with the following text: "${llmDoesNotKnowMessage}" -When crafting your final answer to the user, follow these guidelines: -${makeMarkdownNumberedList(finalAnswerContentNotes)} - Response format: ${makeMarkdownNumberedList(responseFormat)} @@ -145,20 +142,9 @@ ${makeMarkdownNumberedList(searchRequiresRephraseNotes)} - - -You have access to the ${SEARCH_TOOL_NAME} tool. Use the ${SEARCH_TOOL_NAME} tool as follows: -${makeMarkdownNumberedList(searchContentToolNotes)} - -When you search, include metadata about the relevant MongoDB programming language and product. - - - - -You have access to the ${FETCH_PAGE_TOOL_NAME} tool. Use the ${FETCH_PAGE_TOOL_NAME} tool as follows: -${makeMarkdownNumberedList(fetchPageToolNotes)} +You have access to the ${SEARCH_TOOL_NAME} and ${FETCH_PAGE_TOOL_NAME} tools. - +If the ${SEARCH_TOOL_NAME} was used, and it returned "${SEARCH_ALL_FALLBACK_TEXT}", you should say that you did not find an answer on the page provided and that you searched your knowledge base to find an answer. `, } satisfies SystemMessage; diff --git a/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts b/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts index 0ea7fd135..59fd25f8f 100644 --- a/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts +++ b/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts @@ -9,6 +9,7 @@ import { MakeReferenceLinksFunc } from "mongodb-chatbot-server"; import { normalizeUrl } from "mongodb-rag-core/dataSources"; import { wrapTraced } from "mongodb-rag-core/braintrust"; import { Tool, tool } from "mongodb-rag-core/aiSdk"; +import { SEARCH_TOOL_NAME } from "./search"; export const FETCH_PAGE_TOOL_NAME = "fetch_page"; export const SEARCH_ALL_FALLBACK_TEXT = "{fallback_to_search}"; @@ -38,6 +39,20 @@ export interface MakeFetchPageToolParams { pageLengthCutoff?: number; } +const fetchPageToolNotes = [ + "Fetch the entire page content for a given URL.", + `Do not assume that the user wants to use the ${FETCH_PAGE_TOOL_NAME} based on the URL in the Front Matter. The ${FETCH_PAGE_TOOL_NAME} should ONLY be used if the user implies you should look on the page or if the user explicitly provides a URL in their question.`, + `If the user provides multiple URLs in their query, call the ${FETCH_PAGE_TOOL_NAME} once for each URL, and do not call the ${FETCH_PAGE_TOOL_NAME} for the URL in the Front Matter.`, + "Sometimes, when a page is very long, a search will be performed over the page. Therefore, you must also provide a search query to the tool.", + "Do not include URLs in the search query.", + `If the ${FETCH_PAGE_TOOL_NAME} tool returns the string "${SEARCH_ALL_FALLBACK_TEXT}", you MUST immediately call the ${SEARCH_TOOL_NAME} tool.`, + `If the results from the ${FETCH_PAGE_TOOL_NAME} are not sufficient to answer the user's question, you may call the ${SEARCH_TOOL_NAME} tool to find an answer.`, +]; + +function makeMarkdownNumberedList(items: string[]) { + return items.map((item, i) => `${i + 1}. ${item}`).join("\n"); +} + export function makeFetchPageTool({ loadPage, findContent, @@ -47,7 +62,9 @@ export function makeFetchPageTool({ }: MakeFetchPageToolParams): FetchPageTool { return tool({ inputSchema: MongoDbFetchPageToolArgsSchema, - description: "Fetch all content for a specific URL", + description: `Fetch all content for a specific URL. Use it as follows: + +${makeMarkdownNumberedList(fetchPageToolNotes)}`, toModelOutput(output) { return { type: "content", diff --git a/packages/chatbot-server-mongodb-public/src/tools/search.ts b/packages/chatbot-server-mongodb-public/src/tools/search.ts index 0efa569b4..51afeb092 100644 --- a/packages/chatbot-server-mongodb-public/src/tools/search.ts +++ b/packages/chatbot-server-mongodb-public/src/tools/search.ts @@ -56,13 +56,28 @@ export interface MakeSearchToolParams { makeReferences: MakeReferenceLinksFunc; } +const searchContentToolNotes = [ + "Search all of the available MongoDB reference documents for a given user input.", + "You must generate an appropriate search query for a given user input.", + "You are doing this for MongoDB, and all queries relate to MongoDB products.", + `Only generate ONE ${SEARCH_TOOL_NAME} tool call per user message unless there are clearly multiple distinct queries needed to answer the user query.`, +]; + +function makeMarkdownNumberedList(items: string[]) { + return items.map((item, i) => `${i + 1}. ${item}`).join("\n"); +} + export function makeSearchTool({ findContent, makeReferences, }: MakeSearchToolParams): SearchTool { const searchTool: SearchTool = tool({ inputSchema: MongoDbSearchToolArgsSchema, - description: "Search MongoDB content", + description: `Search MongoDB content. Use the ${SEARCH_TOOL_NAME} tool as follows: + +${makeMarkdownNumberedList(searchContentToolNotes)} + +When you search, include metadata about the relevant MongoDB programming language and product.`, toModelOutput(result) { return { From 81758d1a1e839fed2366b0eed250a88a2203f3e9 Mon Sep 17 00:00:00 2001 From: Helen Schawe Date: Wed, 27 Aug 2025 12:01:48 -0500 Subject: [PATCH 06/12] Making progress --- .../src/systemPrompt.ts | 42 +++++++++---------- .../src/tools/fetchPage.ts | 15 +++---- .../src/tools/search.ts | 5 +-- 3 files changed, 27 insertions(+), 35 deletions(-) diff --git a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts index 2fd5ebdee..59f343662 100644 --- a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts +++ b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts @@ -3,6 +3,10 @@ import { mongoDbProducts, mongoDbProgrammingLanguages, } from "mongodb-rag-core/mongoDbMetadata"; +import { + makeMarkdownUnorderedList, + makeMarkdownNumberedList, +} from "mongodb-rag-core/dataSources"; import { SEARCH_TOOL_NAME } from "./tools/search"; import { FETCH_PAGE_TOOL_NAME, @@ -60,24 +64,14 @@ const searchRequiresRephraseNotes = [ "You should also transform the user query into a fully formed question, if relevant.", ]; -const searchContentToolNotes = [ - "Search all of the available MongoDB reference documents for a given user input.", - "You must generate an appropriate search query for a given user input.", - "You are doing this for MongoDB, and all queries relate to MongoDB products.", - `Only generate ONE ${SEARCH_TOOL_NAME} tool call per user message unless there are clearly multiple distinct queries needed to answer the user query.`, -]; - -const fetchPageToolNotes = [ - "Fetch the entire page content for a given URL.", - `Do not assume that the user wants to use the ${FETCH_PAGE_TOOL_NAME} based on the URL in the Front Matter. The ${FETCH_PAGE_TOOL_NAME} should ONLY be used if the user implies you should look on the page or if the user explicitly provides a URL in their question.`, - `If the user provides URLs in their query, ONLY call the ${FETCH_PAGE_TOOL_NAME} for those URLs, and do NOT call the ${FETCH_PAGE_TOOL_NAME} for the URL in the Front Matter.`, - "Sometimes, when a page is very long, a search will be performed over the page. Therefore, you must also provide a search query to the tool.", - "Do not include URLs in the search query.", +const coordinateToolNotes = [ `If the ${FETCH_PAGE_TOOL_NAME} tool returns the string "${SEARCH_ALL_FALLBACK_TEXT}", you MUST immediately call the ${SEARCH_TOOL_NAME} tool.`, + `If the results from the ${FETCH_PAGE_TOOL_NAME} are not sufficient to answer the user's question, you should call the ${SEARCH_TOOL_NAME} tool to find an answer.`, ]; -const finalAnswerContentNotes = [ - `If the ${FETCH_PAGE_TOOL_NAME} tool replies with "${SEARCH_ALL_FALLBACK_TEXT}", you should tell the user you didn't find anything on the page provided and that you searched your knowledge base to find an answer.`, +const toolUseDisclaimers = [ + `If you used the ${FETCH_PAGE_TOOL_NAME} tool, and it returned ${SEARCH_ALL_FALLBACK_TEXT}, in your final answer you should say: "I couldn't use that page to answer your question, so I searched my knowledge base to find an answer."`, + `If the user provided a URL but you had to use ${SEARCH_TOOL_NAME} to find an answer, in your final answer you should say: "I couldn't use that page to answer your question, so I searched my knowledge base to find an answer.`, ]; const importantNote = ` @@ -142,17 +136,23 @@ ${makeMarkdownNumberedList(searchRequiresRephraseNotes)} -You have access to the ${SEARCH_TOOL_NAME} and ${FETCH_PAGE_TOOL_NAME} tools. +You have access to the search_content and fetch_page tools. + + + +${makeMarkdownUnorderedList(coordinateToolNotes)} -If the ${SEARCH_TOOL_NAME} was used, and it returned "${SEARCH_ALL_FALLBACK_TEXT}", you should say that you did not find an answer on the page provided and that you searched your knowledge base to find an answer. + + + + +${makeMarkdownUnorderedList(toolUseDisclaimers)} + + `, } satisfies SystemMessage; -function makeMarkdownNumberedList(items: string[]) { - return items.map((item, i) => `${i + 1}. ${item}`).join("\n"); -} - export const makeMongoDbAssistantSystemPrompt: MakeSystemPrompt = ( customSystemPrompt, customToolDefinitions diff --git a/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts b/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts index 59fd25f8f..9df6bda36 100644 --- a/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts +++ b/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts @@ -6,10 +6,12 @@ import { Reference, } from "mongodb-rag-core"; import { MakeReferenceLinksFunc } from "mongodb-chatbot-server"; -import { normalizeUrl } from "mongodb-rag-core/dataSources"; +import { + normalizeUrl, + makeMarkdownNumberedList, +} from "mongodb-rag-core/dataSources"; import { wrapTraced } from "mongodb-rag-core/braintrust"; import { Tool, tool } from "mongodb-rag-core/aiSdk"; -import { SEARCH_TOOL_NAME } from "./search"; export const FETCH_PAGE_TOOL_NAME = "fetch_page"; export const SEARCH_ALL_FALLBACK_TEXT = "{fallback_to_search}"; @@ -40,19 +42,12 @@ export interface MakeFetchPageToolParams { } const fetchPageToolNotes = [ - "Fetch the entire page content for a given URL.", `Do not assume that the user wants to use the ${FETCH_PAGE_TOOL_NAME} based on the URL in the Front Matter. The ${FETCH_PAGE_TOOL_NAME} should ONLY be used if the user implies you should look on the page or if the user explicitly provides a URL in their question.`, `If the user provides multiple URLs in their query, call the ${FETCH_PAGE_TOOL_NAME} once for each URL, and do not call the ${FETCH_PAGE_TOOL_NAME} for the URL in the Front Matter.`, "Sometimes, when a page is very long, a search will be performed over the page. Therefore, you must also provide a search query to the tool.", "Do not include URLs in the search query.", - `If the ${FETCH_PAGE_TOOL_NAME} tool returns the string "${SEARCH_ALL_FALLBACK_TEXT}", you MUST immediately call the ${SEARCH_TOOL_NAME} tool.`, - `If the results from the ${FETCH_PAGE_TOOL_NAME} are not sufficient to answer the user's question, you may call the ${SEARCH_TOOL_NAME} tool to find an answer.`, ]; -function makeMarkdownNumberedList(items: string[]) { - return items.map((item, i) => `${i + 1}. ${item}`).join("\n"); -} - export function makeFetchPageTool({ loadPage, findContent, @@ -62,7 +57,7 @@ export function makeFetchPageTool({ }: MakeFetchPageToolParams): FetchPageTool { return tool({ inputSchema: MongoDbFetchPageToolArgsSchema, - description: `Fetch all content for a specific URL. Use it as follows: + description: `Fetches the entire page contents for a specific URL. Use this tool as follows: ${makeMarkdownNumberedList(fetchPageToolNotes)}`, toModelOutput(output) { diff --git a/packages/chatbot-server-mongodb-public/src/tools/search.ts b/packages/chatbot-server-mongodb-public/src/tools/search.ts index 51afeb092..e71849f5d 100644 --- a/packages/chatbot-server-mongodb-public/src/tools/search.ts +++ b/packages/chatbot-server-mongodb-public/src/tools/search.ts @@ -11,6 +11,7 @@ import { mongoDbProducts, mongoDbProgrammingLanguageIds, } from "mongodb-rag-core/mongoDbMetadata"; +import { makeMarkdownNumberedList } from "mongodb-rag-core/dataSources"; import { wrapTraced } from "mongodb-rag-core/braintrust"; import { MakeReferenceLinksFunc } from "mongodb-chatbot-server"; @@ -63,10 +64,6 @@ const searchContentToolNotes = [ `Only generate ONE ${SEARCH_TOOL_NAME} tool call per user message unless there are clearly multiple distinct queries needed to answer the user query.`, ]; -function makeMarkdownNumberedList(items: string[]) { - return items.map((item, i) => `${i + 1}. ${item}`).join("\n"); -} - export function makeSearchTool({ findContent, makeReferences, From 24e00231e5a705fa87e5cd30d87149d4cbaabc87 Mon Sep 17 00:00:00 2001 From: Helen Schawe Date: Wed, 27 Aug 2025 12:19:47 -0500 Subject: [PATCH 07/12] Add missing file --- packages/mongodb-rag-core/src/dataSources/index.ts | 1 + .../mongodb-rag-core/src/dataSources/makeMarkdownLists.ts | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 packages/mongodb-rag-core/src/dataSources/makeMarkdownLists.ts diff --git a/packages/mongodb-rag-core/src/dataSources/index.ts b/packages/mongodb-rag-core/src/dataSources/index.ts index dea6c5b65..1962c36ef 100644 --- a/packages/mongodb-rag-core/src/dataSources/index.ts +++ b/packages/mongodb-rag-core/src/dataSources/index.ts @@ -11,3 +11,4 @@ export * from "./MarkdownUrlDataSource"; export * from "./LangchainDocumentLoaderDataSource"; export * from "./CodeOnGithubTextDataSource"; export * from "./normalizeUrl"; +export * from "./makeMarkdownLists"; diff --git a/packages/mongodb-rag-core/src/dataSources/makeMarkdownLists.ts b/packages/mongodb-rag-core/src/dataSources/makeMarkdownLists.ts new file mode 100644 index 000000000..022206bd1 --- /dev/null +++ b/packages/mongodb-rag-core/src/dataSources/makeMarkdownLists.ts @@ -0,0 +1,7 @@ +export function makeMarkdownNumberedList(items: string[]) { + return items.map((item, i) => `${i + 1}. ${item}`).join("\n"); +} + +export function makeMarkdownUnorderedList(items: string[]) { + return items.map((item) => `- ${item}`).join("\n"); +} From 0d1cf60ccff15eafae861d0630b80a98422f7882 Mon Sep 17 00:00:00 2001 From: Helen Schawe Date: Wed, 27 Aug 2025 13:55:31 -0500 Subject: [PATCH 08/12] No matching URL fallback case only --- packages/chatbot-server-mongodb-public/src/systemPrompt.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts index 59f343662..c360beb84 100644 --- a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts +++ b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts @@ -66,12 +66,10 @@ const searchRequiresRephraseNotes = [ const coordinateToolNotes = [ `If the ${FETCH_PAGE_TOOL_NAME} tool returns the string "${SEARCH_ALL_FALLBACK_TEXT}", you MUST immediately call the ${SEARCH_TOOL_NAME} tool.`, - `If the results from the ${FETCH_PAGE_TOOL_NAME} are not sufficient to answer the user's question, you should call the ${SEARCH_TOOL_NAME} tool to find an answer.`, ]; const toolUseDisclaimers = [ - `If you used the ${FETCH_PAGE_TOOL_NAME} tool, and it returned ${SEARCH_ALL_FALLBACK_TEXT}, in your final answer you should say: "I couldn't use that page to answer your question, so I searched my knowledge base to find an answer."`, - `If the user provided a URL but you had to use ${SEARCH_TOOL_NAME} to find an answer, in your final answer you should say: "I couldn't use that page to answer your question, so I searched my knowledge base to find an answer.`, + `If the ${FETCH_PAGE_TOOL_NAME} tool returns ${SEARCH_ALL_FALLBACK_TEXT}, you must tell the user in your final answer: "I couldn't use that page to answer your question, so I searched my knowledge base to find an answer."`, ]; const importantNote = ` From 255c9982fadc2db50ac550c3ab3a4b47f27e5907 Mon Sep 17 00:00:00 2001 From: Helen Schawe Date: Wed, 27 Aug 2025 14:28:40 -0500 Subject: [PATCH 09/12] Minor but impactful wording change --- packages/chatbot-server-mongodb-public/src/systemPrompt.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts index c360beb84..1af1b1ff8 100644 --- a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts +++ b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts @@ -69,7 +69,7 @@ const coordinateToolNotes = [ ]; const toolUseDisclaimers = [ - `If the ${FETCH_PAGE_TOOL_NAME} tool returns ${SEARCH_ALL_FALLBACK_TEXT}, you must tell the user in your final answer: "I couldn't use that page to answer your question, so I searched my knowledge base to find an answer."`, + `If you called the ${FETCH_PAGE_TOOL_NAME} tool and it returned the string "${SEARCH_ALL_FALLBACK_TEXT}", you must tell the user in your final answer: "I couldn't use that page to answer your question, so I searched my knowledge base to find an answer."`, ]; const importantNote = ` From 50cbf6102d8e77d3b0d5ece339178691df4f1827 Mon Sep 17 00:00:00 2001 From: Helen Schawe Date: Wed, 27 Aug 2025 16:51:43 -0500 Subject: [PATCH 10/12] Preface instructions & run all conversation evals --- .../chatbot-server-mongodb-public/src/conversations.eval.ts | 4 ++-- packages/chatbot-server-mongodb-public/src/systemPrompt.ts | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/chatbot-server-mongodb-public/src/conversations.eval.ts b/packages/chatbot-server-mongodb-public/src/conversations.eval.ts index 695cbc287..48ab6c381 100644 --- a/packages/chatbot-server-mongodb-public/src/conversations.eval.ts +++ b/packages/chatbot-server-mongodb-public/src/conversations.eval.ts @@ -64,13 +64,13 @@ async function conversationEval() { ...systemPromptCases, ...internalToolCases, ...customToolCases, - ].filter((c) => c.tags?.includes("fetch_page") || c.tags?.includes("systemPromptAdherence")); + ]; try { // Run the conversation eval const evalResult = await makeConversationEval({ projectName: CONVERSATION_EVAL_PROJECT_NAME, - experimentName: "mongodb-chatbot-feature-fetch-adherence", + experimentName: "mongodb-chatbot-latest", metadata: { description: "Evaluates how well the MongoDB AI Chatbot RAG pipeline works", diff --git a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts index 1af1b1ff8..e82f21e67 100644 --- a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts +++ b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts @@ -138,12 +138,16 @@ You have access to the search_content and fetch_page tools. +Follow these guidelines when using the tools: + ${makeMarkdownUnorderedList(coordinateToolNotes)} +When writing your final answer, provide any necessary disclaimers: + ${makeMarkdownUnorderedList(toolUseDisclaimers)} From bf440dc8cfc889991c465bbda7c33cd533340848 Mon Sep 17 00:00:00 2001 From: Helen Schawe Date: Thu, 28 Aug 2025 11:42:58 -0500 Subject: [PATCH 11/12] Clarify fetch_page vs. search_content confusion --- packages/chatbot-server-mongodb-public/src/systemPrompt.ts | 2 ++ packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts index e82f21e67..baf32b2ed 100644 --- a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts +++ b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts @@ -65,6 +65,8 @@ const searchRequiresRephraseNotes = [ ]; const coordinateToolNotes = [ + `When deciding whether to call ${FETCH_PAGE_TOOL_NAME}, DO NOT ASSUME that the user wants to use the ${FETCH_PAGE_TOOL_NAME} based on the pageUrl in the Front Matter.`, + `The ${FETCH_PAGE_TOOL_NAME} tool should only be used if the user asks you to reference the page they are on (e.g. "Use the page I'm on..."), or if the user explicitly provides a URL in their question.`, `If the ${FETCH_PAGE_TOOL_NAME} tool returns the string "${SEARCH_ALL_FALLBACK_TEXT}", you MUST immediately call the ${SEARCH_TOOL_NAME} tool.`, ]; diff --git a/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts b/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts index 9df6bda36..0fe1ef8e0 100644 --- a/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts +++ b/packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts @@ -42,7 +42,7 @@ export interface MakeFetchPageToolParams { } const fetchPageToolNotes = [ - `Do not assume that the user wants to use the ${FETCH_PAGE_TOOL_NAME} based on the URL in the Front Matter. The ${FETCH_PAGE_TOOL_NAME} should ONLY be used if the user implies you should look on the page or if the user explicitly provides a URL in their question.`, + `Remember - do not assume that the user wants to use the ${FETCH_PAGE_TOOL_NAME} based on the pageUrl in the Front Matter. Only call this tool when specifically instructed to use a certain page.`, `If the user provides multiple URLs in their query, call the ${FETCH_PAGE_TOOL_NAME} once for each URL, and do not call the ${FETCH_PAGE_TOOL_NAME} for the URL in the Front Matter.`, "Sometimes, when a page is very long, a search will be performed over the page. Therefore, you must also provide a search query to the tool.", "Do not include URLs in the search query.", From e129f81d5e52178950c20f85c97848e4c5d73faf Mon Sep 17 00:00:00 2001 From: Helen Schawe <71039431+hschawe@users.noreply.github.com> Date: Thu, 28 Aug 2025 13:44:20 -0500 Subject: [PATCH 12/12] PR review: string interpolation of tool names Co-authored-by: Ben Perlmutter <90647379+mongodben@users.noreply.github.com> --- packages/chatbot-server-mongodb-public/src/systemPrompt.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts index baf32b2ed..d172e10c6 100644 --- a/packages/chatbot-server-mongodb-public/src/systemPrompt.ts +++ b/packages/chatbot-server-mongodb-public/src/systemPrompt.ts @@ -136,7 +136,7 @@ ${makeMarkdownNumberedList(searchRequiresRephraseNotes)} -You have access to the search_content and fetch_page tools. +You have access to the ${SEARCH_TOOL_NAME} and ${FETCH_PAGE_TOOL_NAME} tools.