From 4a531a5b039484b26e1d632d7b5efd578ebbe31e Mon Sep 17 00:00:00 2001 From: James Braza Date: Fri, 26 Sep 2025 13:02:49 -0700 Subject: [PATCH 1/3] Making current evidence and relevant evidence different in status --- src/paperqa/agents/env.py | 14 +++++++++----- src/paperqa/agents/tools.py | 24 ++++++++++++++++-------- tests/test_agents.py | 6 +++--- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/paperqa/agents/env.py b/src/paperqa/agents/env.py index b509ad4a9..3f51ec982 100644 --- a/src/paperqa/agents/env.py +++ b/src/paperqa/agents/env.py @@ -141,6 +141,7 @@ def make_clinical_trial_status( total_clinical_trials: int, relevant_clinical_trials: int, evidence_count: int, + relevant_evidence_count: int, cost: float, ) -> str: return ( @@ -148,15 +149,17 @@ def make_clinical_trial_status( f" | Relevant Papers={relevant_paper_count}" f" | Clinical Trial Count={total_clinical_trials}" f" | Relevant Clinical Trials={relevant_clinical_trials}" - f" | Current Evidence={evidence_count}" + f" | Evidence Count={evidence_count}" + f" | Relevant Evidence={relevant_evidence_count}" f" | Current Cost=${cost:.4f}" ) -# SEE: https://regex101.com/r/L0L5MH/1 +# SEE: https://regex101.com/r/L0L5MH/4 CLINICAL_STATUS_SEARCH_REGEX_PATTERN: str = ( - r"Status: Paper Count=(\d+) \| Relevant Papers=(\d+)(?:\s\|\sClinical Trial" - r" Count=(\d+)\s\|\sRelevant Clinical Trials=(\d+))?\s\|\sCurrent Evidence=(\d+)" + r"Status: Paper Count=(\d+)\s\|\sRelevant Papers=(\d+)" + r"(?:\s\|\sClinical Trial Count=(\d+)\s\|\sRelevant Clinical Trials=(\d+))?" + r"\s\|\sEvidence Count=(\d+)\s\|\sRelevant Evidence=(\d+)" ) @@ -195,7 +198,8 @@ def clinical_trial_status(state: "EnvironmentState") -> str: in getattr(c.text.doc, "other", {}).get("client_source", []) } ), - evidence_count=len(relevant_contexts), + evidence_count=len(state.session.contexts), + relevant_evidence_count=len(relevant_contexts), cost=state.session.cost, ) diff --git a/src/paperqa/agents/tools.py b/src/paperqa/agents/tools.py index adfcd6fa1..83895e147 100644 --- a/src/paperqa/agents/tools.py +++ b/src/paperqa/agents/tools.py @@ -25,11 +25,17 @@ def make_status( - total_paper_count: int, relevant_paper_count: int, evidence_count: int, cost: float + total_paper_count: int, + relevant_paper_count: int, + evidence_count: int, + relevant_evidence_count: int, + cost: float, ) -> str: return ( f"Status: Paper Count={total_paper_count}" - f" | Relevant Papers={relevant_paper_count} | Current Evidence={evidence_count}" + f" | Relevant Papers={relevant_paper_count}" + f" | Evidence Count={evidence_count}" + f" | Relevant Evidence={relevant_evidence_count}" f" | Current Cost=${cost:.4f}" ) @@ -39,7 +45,8 @@ def default_status(state: "EnvironmentState") -> str: return make_status( total_paper_count=len(state.docs.docs), relevant_paper_count=len({c.text.doc.dockey for c in relevant_contexts}), - evidence_count=len(relevant_contexts), + evidence_count=len(state.session.contexts), + relevant_evidence_count=len(relevant_contexts), cost=state.session.cost, ) @@ -60,9 +67,10 @@ class EnvironmentState(BaseModel): ), ) - # SEE: https://regex101.com/r/RmuVdC/1 + # SEE: https://regex101.com/r/RmuVdC/3 STATUS_SEARCH_REGEX_PATTERN: ClassVar[str] = ( - r"Status: Paper Count=(\d+) \| Relevant Papers=(\d+) \| Current Evidence=(\d+)" + r"Status: Paper Count=(\d+)\s\|\sRelevant Papers=(\d+)" + r"\s\|\sEvidence Count=(\d+)\s\|\sRelevant Evidence=(\d+)" ) @computed_field # type: ignore[prop-decorator] @@ -350,7 +358,7 @@ async def gen_answer(self, state: EnvironmentState) -> str: # Use to separate answer from status # NOTE: can match failure to answer or an actual answer ANSWER_SPLIT_REGEX_PATTERN: ClassVar[str] = ( - r" \| " + EnvironmentState.STATUS_SEARCH_REGEX_PATTERN + r"\s\|\s" + EnvironmentState.STATUS_SEARCH_REGEX_PATTERN ) @classmethod @@ -359,7 +367,7 @@ def extract_answer_from_message(cls, content: str) -> str: answer, *rest = re.split( pattern=cls.ANSWER_SPLIT_REGEX_PATTERN, string=content, maxsplit=1 ) - return answer if len(rest) == 4 else "" # noqa: PLR2004 + return answer if len(rest) == 5 else "" # noqa: PLR2004 class Reset(NamedTool): @@ -383,7 +391,7 @@ class Complete(NamedTool): # Use to separate certainty from status CERTAINTY_SPLIT_REGEX_PATTERN: ClassVar[str] = ( - r" \| " + EnvironmentState.STATUS_SEARCH_REGEX_PATTERN + r"\s\|\s" + EnvironmentState.STATUS_SEARCH_REGEX_PATTERN ) NO_ANSWER_PHRASE: ClassVar[str] = "No answer generated." diff --git a/tests/test_agents.py b/tests/test_agents.py index b6e7f116a..25d147948 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -397,11 +397,11 @@ async def test_successful_memory_agent(agent_test_settings: Settings) -> None: " and you have already tried to answer several times," " you can terminate by calling the {complete_tool_name} tool." " The current status of evidence/papers/cost is " - f"{make_status(total_paper_count=0, relevant_paper_count=0, evidence_count=0, cost=0.0)}" # Started 0 # noqa: E501 + f"{make_status(total_paper_count=0, relevant_paper_count=0, evidence_count=0, relevant_evidence_count=0, cost=0.0)}" # Started 0 # noqa: E501 "\n\nTool request message '' for tool calls: paper_search(query='XAI for" " chemical property prediction', min_year='2018', max_year='2024')" f" [id={memory_id}]\n\nTool response message '" - f"{make_status(total_paper_count=2, relevant_paper_count=0, evidence_count=0, cost=0.0)}" # Found 2 # noqa: E501 + f"{make_status(total_paper_count=2, relevant_paper_count=0, evidence_count=0, relevant_evidence_count=0, cost=0.0)}" # Found 2 # noqa: E501 f"' for tool call ID {memory_id} of tool 'paper_search'" ), input=( @@ -412,7 +412,7 @@ async def test_successful_memory_agent(agent_test_settings: Settings) -> None: " and you have already tried to answer several times," " you can terminate by calling the {complete_tool_name} tool." " The current status of evidence/papers/cost is " - f"{make_status(total_paper_count=0, relevant_paper_count=0, evidence_count=0, cost=0.0)}" + f"{make_status(total_paper_count=0, relevant_paper_count=0, evidence_count=0, relevant_evidence_count=0, cost=0.0)}" # noqa: E501 ), output=( "Tool request message '' for tool calls: paper_search(query='XAI for" From dba082b6730e6119ebdf2fb4baf4a08554948bd2 Mon Sep 17 00:00:00 2001 From: James Braza Date: Fri, 26 Sep 2025 13:18:58 -0700 Subject: [PATCH 2/3] Rephrased current to relevant in gen_answer, to be precise --- src/paperqa/agents/tools.py | 2 +- tests/test_agents.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/paperqa/agents/tools.py b/src/paperqa/agents/tools.py index 83895e147..3f9508a91 100644 --- a/src/paperqa/agents/tools.py +++ b/src/paperqa/agents/tools.py @@ -306,7 +306,7 @@ class GenerateAnswer(NamedTool): async def gen_answer(self, state: EnvironmentState) -> str: """ - Generate an answer using current evidence. + Generate an answer using relevant evidence. The tool may fail, indicating that better or different evidence should be found. Aim for at least five pieces of evidence from multiple sources before invoking this tool. diff --git a/tests/test_agents.py b/tests/test_agents.py index 25d147948..a8b045af4 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -837,7 +837,7 @@ def test_tool_schema(agent_test_settings: Settings) -> None: "info": { "name": "gen_answer", "description": ( - "Generate an answer using current evidence.\n\nThe tool may fail," + "Generate an answer using relevant evidence.\n\nThe tool may fail," " indicating that better or different evidence should be" " found.\nAim for at least five pieces of evidence from multiple" " sources before invoking this tool.\nFeel free to invoke this tool" From af792f4a40bd4102ece1d8b5249237eb3da98a7f Mon Sep 17 00:00:00 2001 From: James Braza Date: Fri, 26 Sep 2025 15:12:57 -0700 Subject: [PATCH 3/3] Having best evidence include 'current question', to make it clear --- src/paperqa/agents/tools.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/paperqa/agents/tools.py b/src/paperqa/agents/tools.py index 3f9508a91..788a269af 100644 --- a/src/paperqa/agents/tools.py +++ b/src/paperqa/agents/tools.py @@ -280,7 +280,13 @@ async def gather_evidence(self, question: str, state: EnvironmentState) -> str: ] ) - best_evidence = f" Best evidence(s):\n\n{top_contexts}" if top_contexts else "" + # Include 'current question' because different questions will lead to + # different best evidences being shown + best_evidence = ( + f" Best evidence(s) for the current question:\n\n{top_contexts}" + if top_contexts + else "" + ) if f"{self.TOOL_FN_NAME}_completed" in self.settings.agent.callbacks: await asyncio.gather(