Skip to content

WIP: Indicate resolved crashes in report #1145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions agent/context_analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,12 @@ def execute(self,
trial=self.trial)
prompt = self.handle_invalid_llm_response(final_response)

if context_result:
logger.info(
'Is context analyzer result consistent: %s',
str(context_result.feasible == last_result.crash_result.true_bug),
trial=self.trial)

# Terminate the inspect tool after the analysis is done
self.inspect_tool.terminate()
analysis_result = resultslib.AnalysisResult(
Expand Down
15 changes: 15 additions & 0 deletions benchmark-sets/analysis-tests-2/libsndfile.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
"functions":
- "name": "sf_open"
"params":
- "name": "path"
"type": "bool "
- "name": "mode"
"type": "int"
- "name": "sfinfo"
"type": "bool "
"return_type": "void"
"signature": "SNDFILE * sf_open(const char *, int, SF_INFO *)"
"language": "c"
"project": "libsndfile"
"target_name": "sndfile_fuzzer"
"target_path": "/src/libsndfile/ossfuzz/sndfile_fuzzer.cc"
1 change: 1 addition & 0 deletions experiment/evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ class Result:
driver_fuzz_err: str = dataclasses.field(kw_only=True, default='')
compile_error: str = ''
compile_log: str = ''
crash_status: int = 0

def __post_init__(self, *args, **kwargs): # pylint: disable=unused-argument
if self.is_driver_fuzz_err:
Expand Down
2 changes: 1 addition & 1 deletion report/templates/index/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,7 @@
<a href="sample/{{ bug.benchmark.id|urlencode }}/{{ bug.sample.id }}.html">Sample {{ bug.sample.id }}</a>
</pre>
<span class="p-2 rounded-lg {{ 'bg-red-300 text-black' if bug.sample.result.crashes and not bug.sample.result.is_semantic_error else 'bg-green-300 text-black' }}">
{{ 'False positive' if bug.sample.result.is_semantic_error else 'True positive' }}
{{ 'True Positive' if bug.sample.result.crash_status == 1 else 'False Positive' if bug.sample.result.crash_status == 2 else 'Resolved' if bug.sample.result.crash_status == 3 else 'Unknown' }}
</span>
</div>
{% endfor %}
Expand Down
25 changes: 25 additions & 0 deletions results.py
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,14 @@ def project_language(self) -> str:
"""Project language of the benchmark."""
return self.benchmark.language

@property
def last_analysis_result(self) -> Optional[AnalysisResult]:
"""Last AnalysisResult in trial."""
for result in self.result_history[::-1]:
if isinstance(result, AnalysisResult):
return result
return None

@property
def best_analysis_result(self) -> Optional[AnalysisResult]:
"""Last AnalysisResult in trial, prefer crashed and a non-semantic error."""
Expand Down Expand Up @@ -630,6 +638,21 @@ def log_path(self) -> str:
return result.log_path
return ''

@property
def crash_status(self) -> int:
"""Crashing status of the last analysis result."""
"""0: Unknown, 1: True Positive, 2: False Positive, 3: No crash."""
result = self.last_analysis_result
if not result:
return 0
if not result.crashes:
return 3
if result.crash_context_result:
return 1 if result.crash_context_result.feasible else 2
if result.crash_result:
return 1 if result.crash_result.true_bug else 2
return 0

@property
def is_semantic_error(self) -> bool:
"""Validates if the best AnalysisResult has semantic error."""
Expand Down Expand Up @@ -695,6 +718,8 @@ def to_dict(self) -> dict:
self.is_semantic_error,
'semantic_error':
self.semantic_error,
'crash_status':
self.crash_status,
}


Expand Down