WIP: Indicate resolved crashes in report

agent/context_analyzer.py

@@ -109,6 +109,12 @@ def execute(self,
                    trial=self.trial)
       prompt = self.handle_invalid_llm_response(final_response)
 
+    if context_result:
+      logger.info(
+          'Is context analyzer result consistent: %s',
+          str(context_result.feasible == last_result.crash_result.true_bug),
+          trial=self.trial)
+
     # Terminate the inspect tool after the analysis is done
     self.inspect_tool.terminate()
     analysis_result = resultslib.AnalysisResult(

benchmark-sets/analysis-tests-2/libsndfile.yaml

@@ -0,0 +1,15 @@
+"functions":
+- "name": "sf_open"
+  "params":
+  - "name": "path"
+    "type": "bool "
+  - "name": "mode"
+    "type": "int"
+  - "name": "sfinfo"
+    "type": "bool "
+  "return_type": "void"
+  "signature": "SNDFILE * sf_open(const char *, int, SF_INFO *)"
+"language": "c"
+"project": "libsndfile"
+"target_name": "sndfile_fuzzer"
+"target_path": "/src/libsndfile/ossfuzz/sndfile_fuzzer.cc"

experiment/evaluator.py

@@ -63,6 +63,7 @@ class Result:
   driver_fuzz_err: str = dataclasses.field(kw_only=True, default='')
   compile_error: str = ''
   compile_log: str = ''
+  crash_status: int = 0
 
   def __post_init__(self, *args, **kwargs):  # pylint: disable=unused-argument
     if self.is_driver_fuzz_err:

report/templates/index/index.html

@@ -422,7 +422,7 @@
                                                             <a href="sample/{{ bug.benchmark.id|urlencode }}/{{ bug.sample.id }}.html">Sample {{ bug.sample.id }}</a>
                                                         </pre>
                                                         <span class="p-2 rounded-lg {{ 'bg-red-300 text-black' if bug.sample.result.crashes and not bug.sample.result.is_semantic_error else 'bg-green-300 text-black' }}">
-                                                            {{ 'False positive' if bug.sample.result.is_semantic_error else 'True positive' }}
+                                                            {{ 'True Positive' if bug.sample.result.crash_status == 1 else 'False Positive' if bug.sample.result.crash_status == 2 else 'Resolved' if bug.sample.result.crash_status == 3 else  'Unknown' }}
                                                         </span>
                                                     </div>
                                                     {% endfor %}

results.py

@@ -454,6 +454,14 @@ def project_language(self) -> str:
     """Project language of the benchmark."""
     return self.benchmark.language
 
+  @property
+  def last_analysis_result(self) -> Optional[AnalysisResult]:
+    """Last AnalysisResult in trial."""
+    for result in self.result_history[::-1]:
+      if isinstance(result, AnalysisResult):
+        return result
+    return None
+
   @property
   def best_analysis_result(self) -> Optional[AnalysisResult]:
     """Last AnalysisResult in trial, prefer crashed and a non-semantic error."""
@@ -630,6 +638,21 @@ def log_path(self) -> str:
       return result.log_path
     return ''
 
+  @property
+  def crash_status(self) -> int:
+    """Crashing status of the last analysis result."""
+    """0: Unknown, 1: True Positive, 2: False Positive, 3: No crash."""
+    result = self.last_analysis_result
+    if not result:
+      return 0
+    if not result.crashes:
+      return 3
+    if result.crash_context_result:
+      return 1 if result.crash_context_result.feasible else 2
+    if result.crash_result:
+      return 1 if result.crash_result.true_bug else 2
+    return 0
+
   @property
   def is_semantic_error(self) -> bool:
     """Validates if the best AnalysisResult has semantic error."""
@@ -695,6 +718,8 @@ def to_dict(self) -> dict:
             self.is_semantic_error,
         'semantic_error':
             self.semantic_error,
+        'crash_status':
+            self.crash_status,
     }