Making current evidence and relevant evidence different in status

Author: jamesbraza

src/paperqa/agents/env.py

@@ -141,22 +141,25 @@ def make_clinical_trial_status(
     total_clinical_trials: int,
     relevant_clinical_trials: int,
     evidence_count: int,
+    relevant_evidence_count: int,
     cost: float,
 ) -> str:
     return (
         f"Status: Paper Count={total_paper_count}"
         f" | Relevant Papers={relevant_paper_count}"
         f" | Clinical Trial Count={total_clinical_trials}"
         f" | Relevant Clinical Trials={relevant_clinical_trials}"
-        f" | Current Evidence={evidence_count}"
+        f" | Evidence Count={evidence_count}"
+        f" | Relevant Evidence={relevant_evidence_count}"
         f" | Current Cost=${cost:.4f}"
     )
 
 
-# SEE: https://regex101.com/r/L0L5MH/1
+# SEE: https://regex101.com/r/L0L5MH/4
 CLINICAL_STATUS_SEARCH_REGEX_PATTERN: str = (
-    r"Status: Paper Count=(\d+) \| Relevant Papers=(\d+)(?:\s\|\sClinical Trial"
-    r" Count=(\d+)\s\|\sRelevant Clinical Trials=(\d+))?\s\|\sCurrent Evidence=(\d+)"
+    r"Status: Paper Count=(\d+)\s\|\sRelevant Papers=(\d+)"
+    r"(?:\s\|\sClinical Trial Count=(\d+)\s\|\sRelevant Clinical Trials=(\d+))?"
+    r"\s\|\sEvidence Count=(\d+)\s\|\sRelevant Evidence=(\d+)"
 )
 
 
@@ -195,7 +198,8 @@ def clinical_trial_status(state: "EnvironmentState") -> str:
                 in getattr(c.text.doc, "other", {}).get("client_source", [])
             }
         ),
-        evidence_count=len(relevant_contexts),
+        evidence_count=len(state.session.contexts),
+        relevant_evidence_count=len(relevant_contexts),
         cost=state.session.cost,
     )
 

src/paperqa/agents/tools.py

@@ -25,11 +25,17 @@
 
 
 def make_status(
-    total_paper_count: int, relevant_paper_count: int, evidence_count: int, cost: float
+    total_paper_count: int,
+    relevant_paper_count: int,
+    evidence_count: int,
+    relevant_evidence_count: int,
+    cost: float,
 ) -> str:
     return (
         f"Status: Paper Count={total_paper_count}"
-        f" | Relevant Papers={relevant_paper_count} | Current Evidence={evidence_count}"
+        f" | Relevant Papers={relevant_paper_count}"
+        f" | Evidence Count={evidence_count}"
+        f" | Relevant Evidence={relevant_evidence_count}"
         f" | Current Cost=${cost:.4f}"
     )
 
@@ -39,7 +45,8 @@ def default_status(state: "EnvironmentState") -> str:
     return make_status(
         total_paper_count=len(state.docs.docs),
         relevant_paper_count=len({c.text.doc.dockey for c in relevant_contexts}),
-        evidence_count=len(relevant_contexts),
+        evidence_count=len(state.session.contexts),
+        relevant_evidence_count=len(relevant_contexts),
         cost=state.session.cost,
     )
 
@@ -60,9 +67,10 @@ class EnvironmentState(BaseModel):
         ),
     )
 
-    # SEE: https://regex101.com/r/RmuVdC/1
+    # SEE: https://regex101.com/r/RmuVdC/3
     STATUS_SEARCH_REGEX_PATTERN: ClassVar[str] = (
-        r"Status: Paper Count=(\d+) \| Relevant Papers=(\d+) \| Current Evidence=(\d+)"
+        r"Status: Paper Count=(\d+)\s\|\sRelevant Papers=(\d+)"
+        r"\s\|\sEvidence Count=(\d+)\s\|\sRelevant Evidence=(\d+)"
     )
 
     @computed_field  # type: ignore[prop-decorator]
@@ -350,7 +358,7 @@ async def gen_answer(self, state: EnvironmentState) -> str:
     # Use to separate answer from status
     # NOTE: can match failure to answer or an actual answer
     ANSWER_SPLIT_REGEX_PATTERN: ClassVar[str] = (
-        r" \| " + EnvironmentState.STATUS_SEARCH_REGEX_PATTERN
+        r"\s\|\s" + EnvironmentState.STATUS_SEARCH_REGEX_PATTERN
     )
 
     @classmethod
@@ -359,7 +367,7 @@ def extract_answer_from_message(cls, content: str) -> str:
         answer, *rest = re.split(
             pattern=cls.ANSWER_SPLIT_REGEX_PATTERN, string=content, maxsplit=1
         )
-        return answer if len(rest) == 4 else ""  # noqa: PLR2004
+        return answer if len(rest) == 5 else ""  # noqa: PLR2004
 
 
 class Reset(NamedTool):
@@ -383,7 +391,7 @@ class Complete(NamedTool):
 
     # Use to separate certainty from status
     CERTAINTY_SPLIT_REGEX_PATTERN: ClassVar[str] = (
-        r" \| " + EnvironmentState.STATUS_SEARCH_REGEX_PATTERN
+        r"\s\|\s" + EnvironmentState.STATUS_SEARCH_REGEX_PATTERN
     )
 
     NO_ANSWER_PHRASE: ClassVar[str] = "No answer generated."

tests/test_agents.py

@@ -397,11 +397,11 @@ async def test_successful_memory_agent(agent_test_settings: Settings) -> None:
             " and you have already tried to answer several times,"
             " you can terminate by calling the {complete_tool_name} tool."
             " The current status of evidence/papers/cost is "
-            f"{make_status(total_paper_count=0, relevant_paper_count=0, evidence_count=0, cost=0.0)}"  # Started 0  # noqa: E501
+            f"{make_status(total_paper_count=0, relevant_paper_count=0, evidence_count=0, relevant_evidence_count=0, cost=0.0)}"  # Started 0  # noqa: E501
             "\n\nTool request message '' for tool calls: paper_search(query='XAI for"
             " chemical property prediction', min_year='2018', max_year='2024')"
             f" [id={memory_id}]\n\nTool response message '"
-            f"{make_status(total_paper_count=2, relevant_paper_count=0, evidence_count=0, cost=0.0)}"  # Found 2  # noqa: E501
+            f"{make_status(total_paper_count=2, relevant_paper_count=0, evidence_count=0, relevant_evidence_count=0, cost=0.0)}"  # Found 2  # noqa: E501
             f"' for tool call ID {memory_id} of tool 'paper_search'"
         ),
         input=(
@@ -412,7 +412,7 @@ async def test_successful_memory_agent(agent_test_settings: Settings) -> None:
             " and you have already tried to answer several times,"
             " you can terminate by calling the {complete_tool_name} tool."
             " The current status of evidence/papers/cost is "
-            f"{make_status(total_paper_count=0, relevant_paper_count=0, evidence_count=0, cost=0.0)}"
+            f"{make_status(total_paper_count=0, relevant_paper_count=0, evidence_count=0, relevant_evidence_count=0, cost=0.0)}"  # noqa: E501
         ),
         output=(
             "Tool request message '' for tool calls: paper_search(query='XAI for"