From 6a480c46a81f4b38f477814264108454a7898540 Mon Sep 17 00:00:00 2001 From: Kartikay Date: Wed, 19 Mar 2025 17:29:20 +0530 Subject: [PATCH 1/4] Implement unified line coverage formula --- experiment/coverage.py | 26 ++++++++++++++++++++++++++ experiment/evaluator.py | 6 ++++-- experiment/textcov.py | 20 ++++++++++++++++++++ report/aggregate_coverage_diff.py | 13 ++++++++----- run_all_experiments.py | 5 ++++- stage/execution_stage.py | 13 +++++++++---- 6 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 experiment/coverage.py diff --git a/experiment/coverage.py b/experiment/coverage.py new file mode 100644 index 0000000000..8cc1adc987 --- /dev/null +++ b/experiment/coverage.py @@ -0,0 +1,26 @@ +"""Standard coverage calculation functions to ensure consistency.""" + +from experiment import textcov + + +def calculate_coverage(cov: textcov.Textcov, linked_lines: int) -> float: + """Calculate coverage according to formula: Cov(f) / Linked(f).""" + if not linked_lines: + return 0.0 + return cov.covered_lines / linked_lines + + +def calculate_coverage_improvement( + new_cov: textcov.Textcov, + existing_cov: textcov.Textcov, + union_linked_lines: int +) -> float: + """Calculate coverage improvement: [Cov(f1) - Cov(f0)] / [Linked(f1 ∪ f0)].""" + if not union_linked_lines: + return 0.0 + + # Make a copy to avoid modifying the original + diff_cov = new_cov.copy() + diff_cov.subtract_covered_lines(existing_cov) + + return diff_cov.covered_lines / union_linked_lines diff --git a/experiment/evaluator.py b/experiment/evaluator.py index 67e07c4a6b..7449ed7502 100644 --- a/experiment/evaluator.py +++ b/experiment/evaluator.py @@ -24,7 +24,7 @@ from google.cloud import storage -from experiment import builder_runner, oss_fuzz_checkout, textcov +from experiment import builder_runner, oss_fuzz_checkout, textcov, coverage as coverage_utils from experiment.benchmark import Benchmark from experiment.builder_runner import BuildResult, RunResult from experiment.fuzz_target_error import SemanticCheckResult @@ -465,7 +465,9 @@ def check_target(self, ai_binary, target_path: str) -> Result: run_result.coverage.subtract_covered_lines(existing_textcov) if total_lines and run_result.coverage: - coverage_diff = run_result.coverage.covered_lines / total_lines + union_linked_lines = max(run_result.coverage.total_lines, total_lines) + coverage_diff = coverage_utils.calculate_coverage_improvement( + run_result.coverage, existing_textcov, union_linked_lines) else: dual_logger.log( f'Warning: total_lines == 0 in {generated_oss_fuzz_project}.') diff --git a/experiment/textcov.py b/experiment/textcov.py index 0fc2f531ab..66e9915e9d 100644 --- a/experiment/textcov.py +++ b/experiment/textcov.py @@ -573,3 +573,23 @@ class name specification use single upper case letter for next_arg += '[]' * array_count args.append(next_arg) return args + + def copy(self) -> 'Textcov': + """Create a deep copy of this Textcov oject.""" + new_cov = Textcov() + # Copy all the functions and files + for name, func in self.functions.items(): + new_func = Function(name=func.name) + for content, line in func.lines.items(): + new_func.lines[content] = Line( + contents=line.contents, hit_count=line.hit_count) + new_cov.functions[name] = new_func + + for name, file in self.files.items(): + new_file = File(name=file.name) + for content, line in file.lines.items(): + new_file.lines[content] = Line( + contents=line.contents, hit_count=line.hit_count) + new_cov.files[name] = new_file + + return new_cov diff --git a/report/aggregate_coverage_diff.py b/report/aggregate_coverage_diff.py index acafc4324f..462c7313a3 100644 --- a/report/aggregate_coverage_diff.py +++ b/report/aggregate_coverage_diff.py @@ -28,6 +28,7 @@ from google.cloud import storage from experiment import evaluator, textcov +from experiment import coverage as coverage_utils def compute_coverage_diff(project: str, coverage_links: list[str]): @@ -53,14 +54,16 @@ def compute_coverage_diff(project: str, coverage_links: list[str]): # TODO: skip other functions defined the target. new_textcov.merge(textcov.Textcov.from_file(f)) - new_textcov.subtract_covered_lines(existing_textcov) + # union of linked lines try: - total_lines = coverage_summary['data'][0]['totals']['lines']['count'] + existing_lines = coverage_summary['data'][0]['totals']['lines']['count'] except KeyError: - total_lines = 1 + existing_lines = 0 - return new_textcov.covered_lines / total_lines - #print(f'{project}:', new_textcov.covered_lines / total_lines) + union_linked_lines = max(new_textcov.total_lines, existing_lines) + + return coverage_utils.calculate_coverage_improvement( + new_textcov, existing_textcov, union_linked_lines) def main(): diff --git a/run_all_experiments.py b/run_all_experiments.py index af4f176b26..b88a289156 100755 --- a/run_all_experiments.py +++ b/run_all_experiments.py @@ -34,6 +34,7 @@ from experiment import evaluator, oss_fuzz_checkout, textcov from experiment.workdir import WorkDirs from llm_toolkit import models, prompt_builder +from experiment import coverage as coverage_utils logger = logging.getLogger(__name__) @@ -510,13 +511,15 @@ def _process_total_coverage_gain() -> dict[str, dict[str, Any]]: cov_relative_gain = 0.0 total_lines = max(total_cov.total_lines, total_existing_lines) + union_linked_lines = max(total_cov.total_lines, total_existing_lines) if total_lines: coverage_gain[project] = { 'language': oss_fuzz_checkout.get_project_language(project), 'coverage_diff': - total_cov.covered_lines / total_lines, + coverage_utils.calculate_coverage_improvement( + total_cov, existing_textcov, union_linked_lines), 'coverage_relative_gain': cov_relative_gain, 'coverage_ofg_total_covered_lines': diff --git a/stage/execution_stage.py b/stage/execution_stage.py index 98013881d8..3c3e8be419 100644 --- a/stage/execution_stage.py +++ b/stage/execution_stage.py @@ -21,6 +21,7 @@ from experiment.evaluator import Evaluator from results import BuildResult, Result, RunResult from stage.base_stage import BaseStage +from experiment import coverage as coverage_utils class ExecutionStage(BaseStage): @@ -112,16 +113,20 @@ def execute(self, result_history: list[Result]) -> Result: coverage_percent = 0.0 existing_textcov = evaluator.load_existing_textcov() - run_result.coverage.subtract_covered_lines(existing_textcov) + # Calculate linked lines union using both textcov objects to determine + # the total number of lines that could be covered + union_linked_lines = max(total_lines, existing_textcov.total_lines) - if total_lines: - coverage_diff = run_result.coverage.covered_lines / total_lines + if union_linked_lines: + coverage_diff = coverage_utils.calculate_coverage_improvement( + run_result.coverage, existing_textcov, union_linked_lines) self.logger.info('coverage diff == %s in %s.', coverage_diff, generated_oss_fuzz_project) else: - self.logger.warning('total_lines == 0 in %s', + self.logger.warning('No linked lines found in %s', generated_oss_fuzz_project) coverage_diff = 0.0 + runresult = RunResult( benchmark=benchmark, trial=last_result.trial, From 546036d8985be63e3b36017d2077742891e56655 Mon Sep 17 00:00:00 2001 From: Kartikay Date: Wed, 19 Mar 2025 17:42:43 +0530 Subject: [PATCH 2/4] formater/linted the code and revreted the log message --- experiment/coverage.py | 34 +++++++++++++++---------------- experiment/evaluator.py | 4 +++- experiment/textcov.py | 24 +++++++++++----------- report/aggregate_coverage_diff.py | 7 ++++--- run_all_experiments.py | 6 +++--- stage/execution_stage.py | 4 ++-- 6 files changed, 40 insertions(+), 39 deletions(-) diff --git a/experiment/coverage.py b/experiment/coverage.py index 8cc1adc987..d302e1919c 100644 --- a/experiment/coverage.py +++ b/experiment/coverage.py @@ -4,23 +4,21 @@ def calculate_coverage(cov: textcov.Textcov, linked_lines: int) -> float: - """Calculate coverage according to formula: Cov(f) / Linked(f).""" - if not linked_lines: - return 0.0 - return cov.covered_lines / linked_lines + """Calculate coverage according to formula: Cov(f) / Linked(f).""" + if not linked_lines: + return 0.0 + return cov.covered_lines / linked_lines -def calculate_coverage_improvement( - new_cov: textcov.Textcov, - existing_cov: textcov.Textcov, - union_linked_lines: int -) -> float: - """Calculate coverage improvement: [Cov(f1) - Cov(f0)] / [Linked(f1 ∪ f0)].""" - if not union_linked_lines: - return 0.0 - - # Make a copy to avoid modifying the original - diff_cov = new_cov.copy() - diff_cov.subtract_covered_lines(existing_cov) - - return diff_cov.covered_lines / union_linked_lines +def calculate_coverage_improvement(new_cov: textcov.Textcov, + existing_cov: textcov.Textcov, + union_linked_lines: int) -> float: + """Calculate coverage improvement: [Cov(f1) - Cov(f0)] / [Linked(f1 ∪ f0)].""" + if not union_linked_lines: + return 0.0 + + # Make a copy to avoid modifying the original + diff_cov = new_cov.copy() + diff_cov.subtract_covered_lines(existing_cov) + + return diff_cov.covered_lines / union_linked_lines diff --git a/experiment/evaluator.py b/experiment/evaluator.py index 7449ed7502..2f0854c9de 100644 --- a/experiment/evaluator.py +++ b/experiment/evaluator.py @@ -24,7 +24,9 @@ from google.cloud import storage -from experiment import builder_runner, oss_fuzz_checkout, textcov, coverage as coverage_utils +from experiment import builder_runner +from experiment import coverage as coverage_utils +from experiment import oss_fuzz_checkout, textcov from experiment.benchmark import Benchmark from experiment.builder_runner import BuildResult, RunResult from experiment.fuzz_target_error import SemanticCheckResult diff --git a/experiment/textcov.py b/experiment/textcov.py index 66e9915e9d..479286123d 100644 --- a/experiment/textcov.py +++ b/experiment/textcov.py @@ -579,17 +579,17 @@ def copy(self) -> 'Textcov': new_cov = Textcov() # Copy all the functions and files for name, func in self.functions.items(): - new_func = Function(name=func.name) - for content, line in func.lines.items(): - new_func.lines[content] = Line( - contents=line.contents, hit_count=line.hit_count) - new_cov.functions[name] = new_func - + new_func = Function(name=func.name) + for content, line in func.lines.items(): + new_func.lines[content] = Line(contents=line.contents, + hit_count=line.hit_count) + new_cov.functions[name] = new_func + for name, file in self.files.items(): - new_file = File(name=file.name) - for content, line in file.lines.items(): - new_file.lines[content] = Line( - contents=line.contents, hit_count=line.hit_count) - new_cov.files[name] = new_file - + new_file = File(name=file.name) + for content, line in file.lines.items(): + new_file.lines[content] = Line(contents=line.contents, + hit_count=line.hit_count) + new_cov.files[name] = new_file + return new_cov diff --git a/report/aggregate_coverage_diff.py b/report/aggregate_coverage_diff.py index 462c7313a3..ca55ed1f71 100644 --- a/report/aggregate_coverage_diff.py +++ b/report/aggregate_coverage_diff.py @@ -27,8 +27,8 @@ from google.cloud import storage -from experiment import evaluator, textcov from experiment import coverage as coverage_utils +from experiment import evaluator, textcov def compute_coverage_diff(project: str, coverage_links: list[str]): @@ -62,8 +62,9 @@ def compute_coverage_diff(project: str, coverage_links: list[str]): union_linked_lines = max(new_textcov.total_lines, existing_lines) - return coverage_utils.calculate_coverage_improvement( - new_textcov, existing_textcov, union_linked_lines) + return coverage_utils.calculate_coverage_improvement(new_textcov, + existing_textcov, + union_linked_lines) def main(): diff --git a/run_all_experiments.py b/run_all_experiments.py index b88a289156..55c7392199 100755 --- a/run_all_experiments.py +++ b/run_all_experiments.py @@ -31,10 +31,10 @@ import run_one_experiment from data_prep import introspector from experiment import benchmark as benchmarklib +from experiment import coverage as coverage_utils from experiment import evaluator, oss_fuzz_checkout, textcov from experiment.workdir import WorkDirs from llm_toolkit import models, prompt_builder -from experiment import coverage as coverage_utils logger = logging.getLogger(__name__) @@ -518,8 +518,8 @@ def _process_total_coverage_gain() -> dict[str, dict[str, Any]]: 'language': oss_fuzz_checkout.get_project_language(project), 'coverage_diff': - coverage_utils.calculate_coverage_improvement( - total_cov, existing_textcov, union_linked_lines), + coverage_utils.calculate_coverage_improvement( + total_cov, existing_textcov, union_linked_lines), 'coverage_relative_gain': cov_relative_gain, 'coverage_ofg_total_covered_lines': diff --git a/stage/execution_stage.py b/stage/execution_stage.py index 3c3e8be419..1e4d8dc805 100644 --- a/stage/execution_stage.py +++ b/stage/execution_stage.py @@ -17,11 +17,11 @@ import os from experiment import builder_runner as builder_runner_lib +from experiment import coverage as coverage_utils from experiment import evaluator as evaluator_lib from experiment.evaluator import Evaluator from results import BuildResult, Result, RunResult from stage.base_stage import BaseStage -from experiment import coverage as coverage_utils class ExecutionStage(BaseStage): @@ -123,7 +123,7 @@ def execute(self, result_history: list[Result]) -> Result: self.logger.info('coverage diff == %s in %s.', coverage_diff, generated_oss_fuzz_project) else: - self.logger.warning('No linked lines found in %s', + self.logger.warning('total_lines == 0 in %s', generated_oss_fuzz_project) coverage_diff = 0.0 From 05fba992c7afaa349ab92e8f933cc784bb301bf2 Mon Sep 17 00:00:00 2001 From: Kartikay Date: Tue, 25 Mar 2025 03:28:56 +0530 Subject: [PATCH 3/4] fix: Standardize line coverage calculation formulas Changes: - Add dedicated coverage.py module with standardized implementations - Fix double-subtraction issue in evaluator.py - Use non-destructive operations to preserve original coverage objects - Replace custom copy implementation with Python's built-in deepcopy --- experiment/coverage.py | 4 ++-- experiment/evaluator.py | 3 --- experiment/textcov.py | 19 ++----------------- 3 files changed, 4 insertions(+), 22 deletions(-) diff --git a/experiment/coverage.py b/experiment/coverage.py index d302e1919c..8319ead603 100644 --- a/experiment/coverage.py +++ b/experiment/coverage.py @@ -4,7 +4,7 @@ def calculate_coverage(cov: textcov.Textcov, linked_lines: int) -> float: - """Calculate coverage according to formula: Cov(f) / Linked(f).""" + """Calculates coverage according to formula: Cov(f) / Linked(f).""" if not linked_lines: return 0.0 return cov.covered_lines / linked_lines @@ -13,7 +13,7 @@ def calculate_coverage(cov: textcov.Textcov, linked_lines: int) -> float: def calculate_coverage_improvement(new_cov: textcov.Textcov, existing_cov: textcov.Textcov, union_linked_lines: int) -> float: - """Calculate coverage improvement: [Cov(f1) - Cov(f0)] / [Linked(f1 ∪ f0)].""" + """Calculates coverage improvement: [Cov(f1) - Cov(f0)] / [Linked(f1 ∪ f0)].""" if not union_linked_lines: return 0.0 diff --git a/experiment/evaluator.py b/experiment/evaluator.py index 2f0854c9de..2febdf3924 100644 --- a/experiment/evaluator.py +++ b/experiment/evaluator.py @@ -463,9 +463,6 @@ def check_target(self, ai_binary, target_path: str) -> Result: coverage_percent = 0.0 existing_textcov = self.load_existing_textcov() - if run_result.coverage: - run_result.coverage.subtract_covered_lines(existing_textcov) - if total_lines and run_result.coverage: union_linked_lines = max(run_result.coverage.total_lines, total_lines) coverage_diff = coverage_utils.calculate_coverage_improvement( diff --git a/experiment/textcov.py b/experiment/textcov.py index 479286123d..d82fad753b 100644 --- a/experiment/textcov.py +++ b/experiment/textcov.py @@ -15,6 +15,7 @@ from __future__ import annotations +import copy import dataclasses import json import logging @@ -576,20 +577,4 @@ class name specification use single upper case letter for def copy(self) -> 'Textcov': """Create a deep copy of this Textcov oject.""" - new_cov = Textcov() - # Copy all the functions and files - for name, func in self.functions.items(): - new_func = Function(name=func.name) - for content, line in func.lines.items(): - new_func.lines[content] = Line(contents=line.contents, - hit_count=line.hit_count) - new_cov.functions[name] = new_func - - for name, file in self.files.items(): - new_file = File(name=file.name) - for content, line in file.lines.items(): - new_file.lines[content] = Line(contents=line.contents, - hit_count=line.hit_count) - new_cov.files[name] = new_file - - return new_cov + return copy.deepcopy(self) From ebe46e49a0bc8f9afc8f6cb3b0aedf89b5846062 Mon Sep 17 00:00:00 2001 From: Kartikay Date: Tue, 25 Mar 2025 04:01:23 +0530 Subject: [PATCH 4/4] fix: Remove unnecessary line coverage subtraction in total coverage calculation This change eliminates the redundant subtraction of covered lines from the total coverage calculation, streamlining the process and ensuring accurate coverage metrics. --- run_all_experiments.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/run_all_experiments.py b/run_all_experiments.py index de4a9941b8..8b3eb9f649 100755 --- a/run_all_experiments.py +++ b/run_all_experiments.py @@ -503,12 +503,6 @@ def _process_total_coverage_gain() -> dict[str, dict[str, Any]]: total_existing_lines = sum(lines) total_cov_covered_lines_before_subtraction = total_cov.covered_lines - total_cov.subtract_covered_lines(existing_textcov) - try: - cov_relative_gain = (total_cov.covered_lines / - existing_textcov.covered_lines) - except ZeroDivisionError: - cov_relative_gain = 0.0 total_lines = max(total_cov.total_lines, total_existing_lines) union_linked_lines = max(total_cov.total_lines, total_existing_lines) @@ -520,8 +514,6 @@ def _process_total_coverage_gain() -> dict[str, dict[str, Any]]: 'coverage_diff': coverage_utils.calculate_coverage_improvement( total_cov, existing_textcov, union_linked_lines), - 'coverage_relative_gain': - cov_relative_gain, 'coverage_ofg_total_covered_lines': total_cov_covered_lines_before_subtraction, 'coverage_ofg_total_new_covered_lines':