WIP: Improving Function and Context Analyzer

agent/base_agent.py

@@ -14,6 +14,7 @@
 """The abstract base class for LLM agents in stages."""
 import argparse
 import asyncio
+import json
 import os
 import random
 import re
@@ -370,6 +371,20 @@ def __init__(self,
 
     logger.info('ADK Agent %s created.', self.name, trial=self.trial)
 
+  def get_xml_representation(self, response: Optional[dict]) -> str:
+    """Returns the XML representation of the response."""
+    if not response:
+      return ''
+    # If the response is not a dict, return it as string
+    if not isinstance(response, dict):
+      return str(response)
+    # Now, we wrap items in a dict with xml tags.
+    xml_rep = []
+    for key, value in response.items():
+      xml_obj = f'<{key}>\n{value}\n</{key}>'
+      xml_rep.append(xml_obj)
+    return '\n'.join(xml_rep)
+
   def chat_llm(self, cur_round: int, client: Any, prompt: Prompt,
                trial: int) -> Any:
     """Call the agent with the given prompt, running async code in sync."""
@@ -398,10 +413,14 @@ async def _call():
               self.log_llm_response(final_response)
             elif event.content.parts[0].function_response:
               final_response = event.content.parts[0].function_response.response
+              self.log_llm_response(self.get_xml_representation(final_response))
           elif event.actions and event.actions.escalate:
             error_message = event.error_message
             logger.error('Agent escalated: %s', error_message, trial=self.trial)
 
+      if not final_response:
+        self.log_llm_response('No valid response from LLM.')
+
       return final_response
 
     return self.llm.with_retry_on_error(lambda: asyncio.run(_call()),

agent/context_analyzer.py

@@ -73,13 +73,18 @@ def execute(self,
 
     last_result = result_history[-1]
 
-    if not isinstance(
-        last_result, resultslib.AnalysisResult) or not last_result.crash_result:
-      logger.error(f'Expected last result to be AnalysisResult, got %s.',
+    # Validate that the last result is an AnalysisResult and has a valid crash_result
+    if not isinstance(last_result, resultslib.AnalysisResult):
+      logger.error('Expected last result to be AnalysisResult, got %s.',
                    type(last_result),
                    trial=self.trial)
       return last_result
 
+    if not last_result.crash_result:
+      logger.error('Missing crash_result in the AnalysisResult.',
+                   trial=self.trial)
+      return last_result
+
     context_result = None
 
     # Initialize the ProjectContainerTool for local file search
@@ -99,6 +104,10 @@ def execute(self,
                                      trial=result_history[-1].trial)
       context_result = resultslib.CrashContextResult.from_dict(final_response)
       if context_result:
+        logger.info(
+            'Is context analyzer result consistent: %s',
+            str(context_result.feasible == last_result.crash_result.true_bug),
+            trial=self.trial)
         break
       logger.error('Failed to parse LLM response into CrashContextResult.',
                    trial=self.trial)
@@ -239,31 +248,30 @@ def get_function_implementation(self, project_name: str,
     return response
 
   def report_final_result(self, feasible: bool, analysis: str,
-                          recommendations: str,
+                          source_code_evidence: str, recommendations: str,
                           tool_context: ToolContext) -> dict:
     """
     Provide final result, including the crash feasibility,
         detailed analysis, and any recommendations.
 
     Args:
         feasible (bool): True if the crash is feasible, False otherwise.
-        analysis (str): Detailed analysis and source code evidence showing
+        analysis (str): Detailed analysis showing
                         why the crash is or is not feasible.
+        source_code_evidence (str): Source code evidence supporting the analysis.
+                                    This MUST show the constraints on input variables and why they make the crash feasible or not feasible.
         recommendations (str): Recommendations for modifying the fuzz target to
                         prevent the crash. If the crash is feasible,
                         this should be empty.
 
     Returns:
         This function will not return anything to the LLM.
     """
-    response = f"""
-      <feasible>\n{feasible}\n</feasible>
-      <analysis>\n{analysis}\n</analysis>
-      <recommendations>\n{recommendations}\n</recommendations>
-    """
-    self.log_llm_response(response)
     crash_context_result = resultslib.CrashContextResult(
-        feasible=feasible, analysis=analysis, recommendations=recommendations)
+        feasible=feasible,
+        analysis=analysis,
+        source_code_evidence=source_code_evidence,
+        recommendations=recommendations)
 
     # We have received final result. Instruct the agent to terminate execution.
     # tool_context._invocation_context.end_invocation = True

agent/function_analyzer.py

@@ -106,15 +106,24 @@ def execute(self,
     # Call the agent asynchronously and return the result
     prompt = self._initial_prompt(result_history)
 
-    final_response_text = self.chat_llm(self.round,
-                                        client=None,
-                                        prompt=prompt,
-                                        trial=result_history[-1].trial)
-
-    self.handle_llm_response(final_response_text, result)
+    while self.round < self.max_round:
+      final_response_text = self.chat_llm(self.round,
+                                          client=None,
+                                          prompt=prompt,
+                                          trial=result_history[-1].trial)
+      if final_response_text:
+        self.handle_llm_response(final_response_text, result)
+        break
+
+      # Handle invalid LLM response
+      template_builder = prompt_builder.FunctionAnalyzerTemplateBuilder(
+          self.llm, self.benchmark)
+
+      prompt = self._container_handle_invalid_tool_usage(
+          [self.inspect_tool], self.round, final_response_text,
+          template_builder.build(), template_builder.get_response_format())
 
     self.inspect_tool.terminate()
-
     return result
 
   def _initial_prompt(
@@ -126,7 +135,7 @@ def _initial_prompt(
     builder = prompt_builder.FunctionAnalyzerTemplateBuilder(
         self.llm, self.benchmark)
 
-    prompt = builder.build_prompt()
+    prompt = builder.build_prompt(self.inspect_tool.project_dir)
 
     prompt.append(self.inspect_tool.tutorial())
 

benchmark-sets/analyzer-tests-1/astc-encoder.yaml

@@ -0,0 +1,15 @@
+"functions":
+- "name": "_Z20symbolic_to_physicalRK21block_size_descriptorRK25symbolic_compressed_blockPh"
+  "params":
+  - "name": "bsd"
+    "type": "bool "
+  - "name": "scb"
+    "type": "bool "
+  - "name": "pcb"
+    "type": "bool "
+  "return_type": "void"
+  "signature": "void symbolic_to_physical(const struct block_size_descriptor &, const struct symbolic_compressed_block &, uint8_t *)"
+"language": "c++"
+"project": "astc-encoder"
+"target_name": "fuzz_astc_physical_to_symbolic"
+"target_path": "/src/astc-encoder/Source/Fuzzers/fuzz_astc_physical_to_symbolic.cpp"

benchmark-sets/analyzer-tests-1/bind9.yaml

@@ -0,0 +1,17 @@
+"functions":
+- "name": "dns_zt_asyncload"
+  "params":
+  - "name": "zt"
+    "type": "bool "
+  - "name": "newonly"
+    "type": "bool"
+  - "name": "loaddone"
+    "type": "bool "
+  - "name": "arg"
+    "type": "bool "
+  "return_type": "int"
+  "signature": "isc_result_t dns_zt_asyncload(dns_zt_t *, bool, dns_zt_callback_t *, void *)"
+"language": "c"
+"project": "bind9"
+"target_name": "isc_lex_gettoken_fuzzer"
+"target_path": "/src/bind9/fuzz/isc_lex_gettoken.c"

benchmark-sets/analyzer-tests-1/bluez.yaml

@@ -0,0 +1,21 @@
+"functions":
+- "name": "g_obex_get_req"
+  "params":
+  - "name": "obex"
+    "type": "bool "
+  - "name": "data_func"
+    "type": "bool "
+  - "name": "complete_func"
+    "type": "bool "
+  - "name": "user_data"
+    "type": "bool "
+  - "name": "err"
+    "type": "bool "
+  - "name": "first_hdr_id"
+    "type": "int"
+  "return_type": "int"
+  "signature": "guint g_obex_get_req(GObex *, GObexDataConsumer, GObexFunc, gpointer, GError **, guint, void)"
+"language": "c"
+"project": "bluez"
+"target_name": "fuzz_textfile"
+"target_path": "/src/fuzz_textfile.c"

benchmark-sets/analyzer-tests-1/clamav.yaml

@@ -0,0 +1,25 @@
+"functions":
+- "name": "Lzma2Decode"
+  "params":
+  - "name": ""
+    "type": "bool "
+  - "name": ""
+    "type": "bool "
+  - "name": ""
+    "type": "bool "
+  - "name": ""
+    "type": "bool "
+  - "name": ""
+    "type": "char"
+  - "name": ""
+    "type": "int"
+  - "name": ""
+    "type": "bool "
+  - "name": ""
+    "type": "bool "
+  "return_type": "int"
+  "signature": "SRes Lzma2Decode(Byte *, SizeT *, const Byte *, SizeT *, Byte, ELzmaFinishMode, ELzmaStatus *, ISzAlloc *)"
+"language": "c++"
+"project": "clamav"
+"target_name": "clamav_scanfile_HWP3_fuzzer"
+"target_path": "/src/clamav/fuzz/clamav_scanfile_fuzzer.cpp"

benchmark-sets/analyzer-tests-1/exiv2.yaml

@@ -0,0 +1,15 @@
+"functions":
+- "name": "_ZNK12_GLOBAL__N_113TiffThumbnail4copyERKN5Exiv28ExifDataE"
+  "params":
+  - "name": ""
+    "type": "bool "
+  - "name": ""
+    "type": "bool "
+  - "name": ""
+    "type": "bool "
+  "return_type": "void"
+  "signature": "struct DataBuf (anonymous namespace)::TiffThumbnail::copy(const ExifData &)"
+"language": "c++"
+"project": "exiv2"
+"target_name": "fuzz-read-print-write"
+"target_path": "/src/exiv2/fuzz/fuzz-read-print-write.cpp"

benchmark-sets/analyzer-tests-1/ibmswtpm2.yaml

@@ -0,0 +1,13 @@
+"functions":
+- "name": "TPM2_Create"
+  "params":
+  - "name": "in"
+    "type": "bool "
+  - "name": "out"
+    "type": "bool "
+  "return_type": "int"
+  "signature": "TPM_RC TPM2_Create(Create_In *, Create_Out *)"
+"language": "c++"
+"project": "ibmswtpm2"
+"target_name": "fuzz_tpm_server"
+"target_path": "/src/ibmswtpm2/src/fuzzer.cc"

benchmark-sets/analyzer-tests-1/libfuse.yaml

@@ -0,0 +1,11 @@
+"functions":
+- "name": "af_gb_alloc_data"
+  "params":
+  - "name": "len"
+    "type": "size_t"
+  "return_type": "void"
+  "signature": "char * af_gb_alloc_data(size_t)"
+"language": "c"
+"project": "libfuse"
+"target_name": "fuzz_optparse"
+"target_path": "/src/fuzz_optparse.c"

benchmark-sets/analyzer-tests-1/liblouis.yaml

@@ -0,0 +1,13 @@
+"functions":
+- "name": "lou_getTypeformForEmphClass"
+  "params":
+  - "name": "tableList"
+    "type": "bool "
+  - "name": "emphClass"
+    "type": "bool "
+  "return_type": "short"
+  "signature": "formtype lou_getTypeformForEmphClass(const char *, const char *)"
+"language": "c"
+"project": "liblouis"
+"target_name": "table_fuzzer"
+"target_path": "/src/liblouis/tests/fuzzing/table_fuzzer.cc"

benchmark-sets/analyzer-tests-1/libraw.yaml

@@ -0,0 +1,15 @@
+"functions":
+- "name": "_ZN6LibRaw17crxLoadDecodeLoopEPvi"
+  "params":
+  - "name": "this"
+    "type": "bool "
+  - "name": "img"
+    "type": "bool "
+  - "name": "nPlanes"
+    "type": "int"
+  "return_type": "void"
+  "signature": "void LibRaw::crxLoadDecodeLoop(void *, int)"
+"language": "c++"
+"project": "libraw"
+"target_name": "libraw_cr2_fuzzer"
+"target_path": "/src/libraw_fuzzer.cc"

benchmark-sets/analyzer-tests-1/libsndfile.yaml

@@ -0,0 +1,15 @@
+"functions":
+- "name": "sf_open"
+  "params":
+  - "name": "path"
+    "type": "bool "
+  - "name": "mode"
+    "type": "int"
+  - "name": "sfinfo"
+    "type": "bool "
+  "return_type": "void"
+  "signature": "SNDFILE * sf_open(const char *, int, SF_INFO *)"
+"language": "c"
+"project": "libsndfile"
+"target_name": "sndfile_fuzzer"
+"target_path": "/src/libsndfile/ossfuzz/sndfile_fuzzer.cc"

benchmark-sets/analyzer-tests-1/libsodium.yaml

@@ -0,0 +1,13 @@
+"functions":
+- "name": "argon2_initialize"
+  "params":
+  - "name": "instance"
+    "type": "bool "
+  - "name": "context"
+    "type": "bool "
+  "return_type": "int"
+  "signature": "int argon2_initialize(argon2_instance_t *, argon2_context *)"
+"language": "c++"
+"project": "libsodium"
+"target_name": "secret_key_auth_fuzzer"
+"target_path": "/src/secret_key_auth_fuzzer.cc"

benchmark-sets/analyzer-tests-1/mosh.yaml

@@ -0,0 +1,15 @@
+"functions":
+- "name": "_ZN8Terminal8Emulator6resizeEmm"
+  "params":
+  - "name": "this"
+    "type": "bool "
+  - "name": "s_width"
+    "type": "size_t"
+  - "name": "s_height"
+    "type": "size_t"
+  "return_type": "void"
+  "signature": "void Terminal::Emulator::resize(size_t, size_t)"
+"language": "c++"
+"project": "mosh"
+"target_name": "terminal_parser_fuzzer"
+"target_path": "/src/mosh/src/fuzz/terminal_parser_fuzzer.cc"

benchmark-sets/analyzer-tests-1/quickjs.yaml

@@ -0,0 +1,19 @@
+"functions":
+- "name": "JS_ParseJSON2"
+  "params":
+  - "name": ""
+    "type": "bool "
+  - "name": ""
+    "type": "bool "
+  - "name": ""
+    "type": "size_t"
+  - "name": ""
+    "type": "bool "
+  - "name": ""
+    "type": "int"
+  "return_type": "void"
+  "signature": "JSValue JS_ParseJSON2(JSContext *, const char *, size_t, const char *, int)"
+"language": "c"
+"project": "quickjs"
+"target_name": "fuzz_regexp"
+"target_path": "/src/quickjs/fuzz/fuzz_regexp.c"

benchmark-sets/analyzer-tests-1/vlc.yaml

@@ -0,0 +1,13 @@
+"functions":
+- "name": "vlm_New"
+  "params":
+  - "name": "libvlc"
+    "type": "bool "
+  - "name": "psz_vlmconf"
+    "type": "bool "
+  "return_type": "void"
+  "signature": "vlm_t * vlm_New(libvlc_int_t *, const char *)"
+"language": "c"
+"project": "vlc"
+"target_name": "vlc-demux-libfuzzer"
+"target_path": "/src/vlc/test/vlc-demux-libfuzzer.c"