Moved used_images to only be opted into via summary_json_multimodal_system_prompt

Author: jamesbraza

README.md

@@ -751,6 +751,12 @@ When creating contextual summaries on a given chunk (a `Text`),
 the summary LLM is passed both the chunk's text and the chunk's associated media,
 but the output contextual summary itself remains text-only.
 
+If you would like,
+specifying the prompt `paperqa.prompts.summary_json_multimodal_system_prompt`
+to `prompt.summary_json_system`
+will include a `used_images` flag attributing
+usage of images in any contextual summarizations.
+
 ### Using External DB/Vector DB and Caching
 
 You may want to cache parsed texts and embeddings in an external database or file.

src/paperqa/prompts.py

@@ -112,7 +112,19 @@
     " Your summary, combined with many others,"
     " will be given to the model to generate an answer."
     " Respond with the following JSON format:"
-    '\n\n{{\n  "summary": "...",\n  "relevance_score": 0-10,\n  "used_images"\n}}'
+    '\n\n{{\n  "summary": "...",\n  "relevance_score": 0-10\n}}'
+    "\n\nwhere `summary` is relevant information from the text - {summary_length} words."
+    " `relevance_score` is an integer 0-10 for the relevance of `summary` to the question."
+    "\n\nThe excerpt may or may not contain relevant information."
+    " If not, leave `summary` empty, and make `relevance_score` be 0."
+)
+summary_json_multimodal_system_prompt = (
+    "Provide a summary of the relevant information"
+    " that could help answer the question based on the excerpt."
+    " Your summary, combined with many others,"
+    " will be given to the model to generate an answer."
+    " Respond with the following JSON format:"
+    '\n\n{{\n  "summary": "...",\n  "relevance_score": 0-10,\n  "used_images": "..."\n}}'
     "\n\nwhere `summary` is relevant information from the text - {summary_length} words."
     " `relevance_score` is an integer 0-10 for the relevance of `summary` to the question."
     " `used_images` is a boolean flag indicating"

tests/test_paperqa.py

@@ -61,7 +61,7 @@
     llm_parse_json,
     map_fxn_summary,
 )
-from paperqa.prompts import CANNOT_ANSWER_PHRASE
+from paperqa.prompts import CANNOT_ANSWER_PHRASE, summary_json_multimodal_system_prompt
 from paperqa.prompts import qa_prompt as default_qa_prompt
 from paperqa.readers import PDFParserFn, parse_image, read_doc
 from paperqa.settings import AsyncContextSerializer
@@ -1592,6 +1592,7 @@ async def test_images(stub_data_dir: Path) -> None:
     # We don't support image embeddings yet, so disable embedding
     settings.answer.evidence_retrieval = False
     settings.parsing.defer_embedding = True
+    settings.prompts.summary_json_system = summary_json_multimodal_system_prompt
 
     docs = Docs()
     districts_docname = await docs.aadd(
@@ -1634,6 +1635,7 @@ async def test_images_corrupt(stub_data_dir: Path) -> None:
     # We don't support image embeddings yet, so disable embedding
     settings.answer.evidence_retrieval = False
     settings.parsing.defer_embedding = True
+    settings.prompts.summary_json_system = summary_json_multimodal_system_prompt
 
     docs = Docs()
     districts_docname = await docs.aadd(