diff --git a/agent/prototyper.py b/agent/prototyper.py index 70fb4ad754..d9a9217f88 100644 --- a/agent/prototyper.py +++ b/agent/prototyper.py @@ -21,6 +21,7 @@ from datetime import timedelta from typing import Optional +from helper.error_classifier import BuildErrorClassifier import logger from agent.base_agent import BaseAgent from data_prep import project_targets @@ -367,15 +368,54 @@ def _generate_prompt_from_build_result( # Preference 7: New fuzz target + both `build.sh`s cannot compile. No need # to mention the default build.sh. # return build_result - builder = prompt_builder.PrototyperFixerTemplateBuilder( - model=self.llm, - benchmark=build_result.benchmark, - build_result=build_result, - compile_log=compile_log, - initial=prompt.get()) - prompt = builder.build(example_pair=[], - project_dir=self.inspect_tool.project_dir) - return build_result, prompt + rag_enabled = False + try: + rag_enabled = bool(getattr(self, 'args', None)) and bool(getattr(self.args, 'rag_classifier', False)) + except Exception: + rag_enabled = False + if rag_enabled: + # Use RAG-based classifier to build a targeted prompt. + error_classifier = BuildErrorClassifier("helper/error_patterns.yaml") + classification = error_classifier.classify(compile_log) + logger.debug("=== Compilation Log Start ===\n%s\n=== Compilation Log End ===", compile_log, trial=build_result.trial) + + if classification: + logger.info("RAG match: identified build error type %s", classification["type"], trial=build_result.trial) + builder = prompt_builder.PrototyperErrorClassifierTemplateBuilder( + model=self.llm, + benchmark=build_result.benchmark, + build_result=build_result, + compile_log=compile_log, + error_classifier=error_classifier, + initial=prompt.get() + ) + prompt = builder.build(project_dir=self.inspect_tool.project_dir) + return build_result, prompt + + # If RAG could not classify, fall back to generic fixer template. + logger.warning("RAG match: classification failed, no error type matched", trial=build_result.trial) + builder = prompt_builder.PrototyperFixerTemplateBuilder( + model=self.llm, + benchmark=build_result.benchmark, + build_result=build_result, + compile_log=compile_log, + initial=prompt.get() + ) + prompt = builder.build(example_pair=[], project_dir=self.inspect_tool.project_dir) + return build_result, prompt + + else: + # RAG disabled -> always use the generic fixer template. + logger.info("RAG classifier disabled (no --rag-classifier flag); using FixerTemplateBuilder.", trial=build_result.trial) + builder = prompt_builder.PrototyperFixerTemplateBuilder( + model=self.llm, + benchmark=build_result.benchmark, + build_result=build_result, + compile_log=compile_log, + initial=prompt.get() + ) + prompt = builder.build(example_pair=[], project_dir=self.inspect_tool.project_dir) + return build_result, prompt def _container_handle_conclusion(self, cur_round: int, response: str, build_result: BuildResult, diff --git a/ci/k8s/pr-exp.yaml b/ci/k8s/pr-exp.yaml index e14ec68ea1..46fd09b808 100644 --- a/ci/k8s/pr-exp.yaml +++ b/ci/k8s/pr-exp.yaml @@ -48,7 +48,7 @@ spec: name: results-volume env: - name: LLM_NUM_EXP - value: '40' + value: '20' - name: LLM_NUM_EVA value: '10' - name: VERTEX_AI_LOCATIONS diff --git a/helper/error_classifier.py b/helper/error_classifier.py new file mode 100644 index 0000000000..a6d40c43d6 --- /dev/null +++ b/helper/error_classifier.py @@ -0,0 +1,51 @@ +import re +import yaml + +class BuildErrorClassifier: + def __init__(self, error_db_path: str): + with open(error_db_path, 'r') as f: + self.error_db = yaml.safe_load(f) + + def classify(self, compile_log: str) -> dict | None: + for error_type, data in self.error_db.items(): + for pattern in data.get("patterns", []): + if re.search(pattern, compile_log, re.IGNORECASE): + return { + "type": error_type, + "good": data.get("good", []), + "bad": data.get("bad", []), + } + return None + + def _find_first_error_msg(self, compile_log: str) -> str | None: + match = re.search(r"(.*?)", compile_log, re.DOTALL) + if match: + compile_log = match.group(1).strip() + else: + return None + + lines = compile_log.splitlines() + for i, line in enumerate(lines): + if any(kw in line.lower() for kw in ('error:', 'fatal error', 'undefined reference')): + return '\n'.join(lines[i:]) + return None + + def trim_and_classify_err_msg(self, compile_log:str) -> dict | None: + compile_log = self._find_first_error_msg(compile_log) + if not compile_log: + return None + for error_type, data in self.error_db.items(): + for pattern in data.get("patterns", []): + try: + match = re.search(pattern, compile_log, re.IGNORECASE) + except Exception: + print(f"Error with pattern: {pattern}") + continue + if match: + return { + "type": error_type, + "trimmed_msg": compile_log.strip()} + return { + "type": "unknown", + "trimmed_msg": compile_log.strip()} + diff --git a/helper/error_patterns.yaml b/helper/error_patterns.yaml new file mode 100644 index 0000000000..16effc7454 --- /dev/null +++ b/helper/error_patterns.yaml @@ -0,0 +1,242 @@ +INCLUDE ERROR: + patterns: + - "fatal error: .*: No such file or directory" + - "error: no such file or directory: '.*'" + - ": .*: No such file or directory" + - "fatal error: '.*' file not found" + - "cannot find include file" + - "header file not found" + - "file not found" + - "include.*No such file" + - "error: include file '.*' not found" + - "'[A-Za-z0-9_]+\\.h': No such file or directory" + - "fatal error: cannot open file '.*'" + - "This header is only to be used internally" + - "forward declaration of '.*'" + - "cannot open source file" + good: + - "Suggest the correct `#include` directive with the accurate path, considering relative paths from the fuzz target's location." + - "Suggest adding the necessary include directory to the compiler's search path (e.g., `-I/path/to/include`)." + - "Leverage umbrella headers that might provide necessary include paths for other project headers." + - "Remove unnecessary includes of internal headers that are not exposed by the public API." + - "If the symbol is from a standard library, suggest the correct standard header." + - "If a symbol belongs to a specific library, ensure the library is linked in the build script." + - "If a header file is not found, suggest adding an include search path to the build script using `-I`." + - "Suggest using include-what-you-use for better include management." + - "If multiple headers could provide the symbol, prioritize the most specific or commonly used one." + bad: + - "Not verifying if a header file exists in the standard include paths or if it requires a specific include path setting in the build script." + - "Using absolute paths when relative paths would be more appropriate and portable." + - "Adding unnecessary or irrelevant includes." + - "Removing include paths without ensuring the header is accessible." + - "Insisting on including internal headers that are not meant for direct inclusion." + - "Assuming header file existence based on naming conventions." + - "Not recognizing that the compiler's include search path might differ from the project's root directory." + - "Copying headers to the fuzz target directory as a workaround." + - "Including source files (`.c`, `.cpp`) directly, leading to potential linker errors." + - "Not considering that the required header might be indirectly included through another header, leading to redundant includes." + +SYNTACTIC ERROR: + patterns: + - 'error: expected .*' + - 'expected .* before' + - 'error: stray '' .* '' in program' + - 'syntax error' + - 'parse error' + - 'missing ''\;''' + - 'implicit conversion changes signedness' + - 'error: assigning to '' .* '' from incompatible type '' .* ''' + - 'error: no member named '' .* '' in '' .* ''' + - 'no matching function for call to' + - 'candidate function not viable' + - 'requires \d+ arguments, but \d+ were provided' + - 'incompatible pointer to .* conversion' + - 'no type named .* in namespace .*' + - 'is a private member of' + - 'expected declaration or statement at end of input' + - 'unterminated string literal' + - 'expected identifier or ''\('' before .*' + - 'expected unqualified-id' + - 'use of undeclared identifier '' .* ''' + - 'error: expected parameter declarator' + - 'error: expected ''\)''' + - 'invalid conversion from' + - 'incomplete type .* named in nested name specifier' + - 'unknown type name .*' + - 'redefinition of .*' + - 'no member named .* in .*' + - 'non-void function does not return a value' + - 'forward declaration of .*' + - 'candidate constructor .* not viable' + - 'call to undeclared function' + - 'ISO C99 and later do not support implicit function declarations' + - 'error: krb5\.h included before k5-int\.h' + - 'This header is only to be used internally to libarchive\.' + - 'class member cannot be redeclared' + - 'expected member name or ''\;'' after declaration specifiers' + good: + - "Remove code that accesses non-existent members of structs, simplifying logic if necessary." + - "Provide correct function prototypes if missing, including `extern \"C\"` if needed." + - "If the error involves a macro, explain its correct usage with concrete, minimal, working examples." + - "Ensure proper usage of parentheses, braces, and semicolons." + - "Check for mismatched types or incorrect function arguments." + - "Replace undefined constants or macros with valid alternatives if available, or remove them if their usage is incorrect." + - "Use code search tools to locate the definition of undefined symbols within the project's source code and include the correct header file." + - "If the error involves templates or generics, verify correct instantiation." + - "Remove redundant definitions of functions or variables if a 'redefinition' error occurs." + bad: + - "Renaming functions or variables without understanding their purpose." + - "Making assumptions about type definitions or function signatures." + - "Incorrectly assuming the existence of members in structs based on similar data structures." + - "Failing to recognize that certain constants or macros might be tied to specific data types or contexts and cannot be used interchangeably." + - "Relying solely on assumptions or incomplete knowledge of the project's structure to guess the header file." + - "Including incorrect header files based on partial matches or similar symbol names." + - "Not recognizing that certain types might be defined within the project but not exposed through the main public header files." + +UNDEFINED REFERENCE ERROR: + patterns: + - "undefined reference to `.*'" + - "undefined reference to '.*'" + - "symbol not found" + - "unresolved external symbol" + - "linker error: symbol undefined" + - "error: undefined reference to `.*`" + - "error: use of undeclared identifier '.*'" + good: + - "Correctly identify the undefined symbol and suggest including the header file containing its declaration." + - "If the symbol is in a library, verify that the library is linked correctly and in the proper order." + - "Suggest adding the `extern \"C\"` linkage specifier to a fuzz target function if it's missing and the linker cannot find the function." + - "Verify function signature compatibility between declaration and definition." + - "If the symbol is a class member, ensure the class is fully defined and accessible." + - "If the symbol is in a namespace, ensure the namespace is correctly used." + - "If the existing fuzz target works correctly, try to replicate its linking approach." + - "If the undefined reference is to a function in an external library, ensure the library is linked correctly in the build script." + - "If explicit linking fails, revert to using build system variables (like `$LIB_FUZZING_ENGINE`) for linking the fuzzing engine library." + bad: + - "Forward declaring a function without providing its definition." + - "Suggesting alternative functions without ensuring they have the required functionality." + - "Providing an incorrect definition for the undefined symbol." + - "Confusing free functions with member functions or static methods." + - "Not considering the possibility of missing library linkages." + - "Suggesting adding function prototypes without including the necessary header files or linking the libraries where the functions are defined." + - "Suggesting changes to compiler or linker flags when restricted to modifying only the source code or build script." + - "Failing to recognize that the undefined reference to the fuzz target entry point is a fundamental requirement and cannot be removed." + +LINKER ERROR: + patterns: + - 'multiple definition of' + - 'ld: duplicate symbol' + - 'conflicting types for' + - 'linking failed' + - 'ld returned 1 exit status' + - 'collect2: error: ld returned .* exit status' + - 'linker command failed with exit code' + - 'cannot find -l.*' + - 'no such file or directory: ''\$LIB_FUZZING_ENGINE''' + - 'relocation overflowed' + - 'error: relocation truncated to fit' + - 'first defined here' + - 'no such file or directory: .*\.a' + - '/usr/bin/ld: cannot find -ljsoncpp' + good: + - "Check for inconsistencies in symbol declarations and definitions across different files." + - "Ensure that the linker knows where to find the built library using `-L`." + - "Link the library using `-l` followed by the library name." + - "If the library has dependencies, ensure those are also built and linked." + - "If dynamic linking fails, try relying on statically linked libraries if they are available." + - "Inspect the build system configuration (CMakeLists.txt, Makefile.am) to understand how libraries are linked." + - "Combine wildcard and explicit linking to ensure all necessary libraries are included." + - "Place the C++ standard library and other system libraries after other libraries in the linker command." + bad: + - "Suggesting including `.c/.cpp` files directly, leading to multiple definition errors." + - "Misunderstanding the one definition rule." + - "Incorrectly stating that the linker is unable to find specific libraries when the error message doesn't mention those libraries." + - "Assuming the linker needs dynamic versions of libraries when static versions are already linked." + - "Not providing a concrete solution for missing library paths, such as using `-L`." + +BUILD CONFIGURATION ERROR: + patterns: + - 'make: \*\*\* No rule to make target .*' + - 'CMake Error:.*' + - 'CMake was unable to find a build program' + - 'CMAKE_(C|CXX)_COMPILER not set' + - 'ninja: error: loading ''build\.ninja'': No such file or directory' + - 'The source directory .* does not appear to contain CMakeLists\.txt' + - 'Policy CMP\d+ is not set' + - 'The OLD behavior for policy CMP\d+ will be removed' + - 'This warning is for project developers\. *Use -Wno-dev to suppress it' + - '/src/build\.sh: line \d+: syntax error' + - 'sed: -e expression #1, char \d+: (Invalid content of \{\}|extra characters after command)' + - 'sed: can''t read.*No such file or directory' + - '/src/build\.sh: line \d+: unbound variable' + - 'unbound variable' + - 'configure: error: .* not found' + - 'configure: WARNING: .*' + - 'autoreconf: ''configure\.(ac|in)'' is required' + - 'Could not find a package configuration file provided by .*' + - 'CMake Error at CMakeLists\.txt:\d+ \(find_package\)' + - 'By not providing "Find.*\.cmake" in CMAKE_MODULE_PATH.*' + - 'debconf: delaying package configuration, since apt-utils is not installed' + - '/src/build\.sh: line \d+: fuzzer/CMakeLists\.txt: No such file or directory' + - 'WARNING: png library not available - no png\.h' + - '\./aom_configure: No such file or directory' + - 'DWARF error: invalid or unhandled FORM value.*' + - 'clang\+\+: error: no such file or directory: ''\$LIB_FUZZING_ENGINE''' + - 'configure: error: .* (required|dependency) .* (not found|not available|missing)' + - 'configure: error: .* but required .* (library|package).* not available' + - 'configure: error: .* requires .* (but it is not installed|not found|not available)' + good: + - "Inspect build system config files (Makefile, CMakeLists.txt, meson.build, configure.ac)." + - "Verify build rules and target dependencies are correctly defined and ordered." + - "Check for missing dependencies/incorrect paths; install exact packages (e.g., libssl-dev, zlib1g-dev)." + - "Clean build dir and rebuild (e.g., rm -rf build && mkdir build)." + - "Set compiler/linker flags via build system (CMAKE_C_FLAGS, target_link_libraries)." + - "Check environment variables (PKG_CONFIG_PATH, CMAKE_PREFIX_PATH, LD_LIBRARY_PATH)." + - "Leverage official project scripts (oss-fuzz.sh, build.sh) when available." + bad: + - "Suggesting modifications to CMake commands or flags when the LLM is restricted to modifying only the source code or a limited build script." + - "Removing the sourcing of configure.sh without understanding its purpose." + - "Suggesting copying the fuzz target source file to the current directory as a solution to a 'no such file or directory' error." + - "Suggesting setting CFLAGS and CXXFLAGS before the configure step, potentially interfering with the configure process." + - "Incorrectly assuming the location of necessary files, the fuzz target is modified in place from the original fuzz target." + - "Using repeated or incorrect sed commands on build files, causing accumulated or broken changes." + - "Overcomplicating build processes when official project script commands suffice." + - "Hardcoding invalid include paths, flags, or library names." + + +CORRUPTED CODE ERROR: + patterns: + - "invalid preprocessing directive" + - "unexpected token" + - "unrecognized input" + - "junk after number" + - "unclosed comment" + - "unterminated comment" + - "unexpected end of file" + - "illegal start of expression" + - "missing terminating ' character" + - "missing terminating \" character" + - "segmentation fault" + - "corrupted" + good: + - "If the code is severely corrupted, suggest reverting to a previous working version." + - "If parts are salvageable, suggest specific changes to fix corrupted sections while preserving logic." + - "Identify and explain the reasons for the corruption (e.g., incorrect merge, bad generation)." + - "Prioritize preserving the original intent and structure of the code." + - "Test suggested changes to ensure they restore functionality." + - "If the corruption is due to missing code, try to generate the missing parts based on context." + - "If the code is partially generated, ensure consistency and correctness of generated parts." + - "Check for logical errors or inconsistencies introduced by the corruption." + - "If the corruption is related to data structures, verify their correctness and consistency." + - "If the cause is unclear, suggest debugging techniques to identify the corrupted sections." + bad: + - "Replacing the entire code with generic placeholder code." + - "Introducing new errors or making the code even more corrupted." + - "Not attempting to understand or preserve the original logic." + - "Failing to test changes or verify they improve the situation." + - "Providing inadequate reasoning that doesn't justify changes." + - "Making assumptions about the intended functionality of the corrupted code." + - "Not recognizing common corruption patterns (e.g., incorrect indentation, missing braces)." + - "Applying generic fixes without understanding the specific corruption." + - "Not considering the context of the corrupted code within the larger project." + - "Ignoring or dismissing the corruption without attempting a fix." diff --git a/llm_toolkit/prompt_builder.py b/llm_toolkit/prompt_builder.py index 3319444453..b3b9d48b25 100644 --- a/llm_toolkit/prompt_builder.py +++ b/llm_toolkit/prompt_builder.py @@ -21,6 +21,7 @@ from abc import abstractmethod from typing import Any, Optional, Tuple +from helper.error_classifier import BuildErrorClassifier import jinja2 from data_prep import introspector, project_targets @@ -671,6 +672,48 @@ def build(self, return self._prompt +class PrototyperErrorClassifierTemplateBuilder(PrototyperTemplateBuilder): + def __init__(self, model, benchmark, build_result, compile_log, error_classifier: BuildErrorClassifier, initial=None): + super().__init__(model, benchmark, DEFAULT_TEMPLATE_DIR, initial) + self.build_result = build_result + self.compile_log = compile_log + self.error_classifier = error_classifier + self.priming_template_file = self._find_template(self.agent_templare_dir, + 'prototyper-error-classifier.txt') + + def build(self, project_dir='') -> prompts.Prompt: + classification = self.error_classifier.classify(self.compile_log) + + error_type = classification["type"] if classification else "UNKNOWN" + function_signature = self.benchmark.function_signature + fuzz_target_source = self.build_result.fuzz_target_source + binary_path = os.path.join('/out', self.benchmark.target_name) + + # Handle build script formatting + if self.build_result.build_script_source: + build_text = (f'\n{self.build_result.build_script_source}\n') + else: + build_text = 'Build script reuses `/src/build.bk.sh`.' + + # Format tips section + if classification: + good_lines = '\n'.join(f"- {line}" for line in classification["good"]) + bad_lines = '\n'.join(f"- {line}" for line in classification["bad"]) + tips = f"What to do (✓):\n{good_lines}\n\nWhat to avoid (✗):\n{bad_lines}" + else: + tips = "No specific suggestions found. Analyze the error log and apply best practices." + + # Load and format template + prompt = self._get_template(self.priming_template_file) + prompt = prompt.replace('{FUZZ_TARGET_SOURCE}', fuzz_target_source) + prompt = prompt.replace('{BUILD_TEXT}', build_text) + prompt = prompt.replace('{COMPILE_LOG}', self.compile_log) + prompt = prompt.replace('{FUNCTION_SIGNATURE}', function_signature) + prompt = prompt.replace('{PROJECT_DIR}', project_dir) + prompt = prompt.replace('{TIPS}', tips) + + self._prompt.append(prompt) + return self._prompt class CoverageAnalyzerTemplateBuilder(PrototyperTemplateBuilder): """Builder specifically targeted C (and excluding C++).""" diff --git a/prompts/agent/prototyper-error-classifier.txt b/prompts/agent/prototyper-error-classifier.txt new file mode 100644 index 0000000000..4999e24617 --- /dev/null +++ b/prompts/agent/prototyper-error-classifier.txt @@ -0,0 +1,44 @@ +Failed to build fuzz target. Here is the fuzz target, build script, and compilation output: + + +{FUZZ_TARGET_SOURCE} + +{BUILD_TEXT} + +{COMPILE_LOG} + + +You are a careful, verification-first build fixer. +Your job is to identify the earliest blocking error and produce the minimal, correct changes to make the target compile. + +PINNED (read before doing anything) +- TIPS are **binding**. You must apply relevant ✓ items from **TIPS (binding)** below and avoid ✗ items. + +Rules (follow strictly): +1) Ground everything in evidence. Do NOT guess. Confirm every assumption (paths, headers, libraries, symbols) with Bash commands before changing code or the build script. +2) Respect the build system. Do not hand-tune environment/global flags outside the provided build script. Work through the build files (CMake/Make/…). +3) Fix the current stage only. If it’s a compile error, don’t propose link-only fixes, and vice versa. +4) Prefer minimal, surgical edits. Do not delete unrelated code. Keep the fuzz target structure intact and use the fuzzed input (`const uint8_t* data, size_t size`) to exercise the target. +5) Use existing working patterns. If similar files/targets show a working approach, mirror that configuration rather than inventing new flows. +6) Interpret file-not-found precisely: + - If it’s a header, consider include correctness and include paths. + - If it’s a source/object/archive (`.c/.cc/.o/.a`), treat it as a build path/configuration issue, not a header/include issue. +7) Explain briefly why your fix works (one or two sentences max). No long essays. + +YOU MUST first analyze the error messages with the fuzz target and the build script carefully to identify the root cause. +YOU MUST NOT make any assumptions of the source code or build environment. Always confirm assumptions with source code evidence, obtained via Bash commands. + +Once you are absolutely certain of the error root cause: +- Provide the FULL SOURCE CODE of the corrected fuzz target. +- If `/src/build.bk.sh` is insufficient, also provide the FULL SOURCE CODE of the updated build script. + +TIPS (binding; follow ✓ and avoid ✗): +{TIPS} + +Focus on writing a compilable fuzz target that calls the function-under-test {FUNCTION_SIGNATURE}. +Coverage and bug finding are NOT priorities now; successful compilation and correctness are. + +Process (do this in order): +A) Diagnose the **earliest blocking error** from the log. +B) Run only minimal verification commands to confirm the cause (e.g., `ls`, `grep`, `pkg-config --cflags --libs`, `cmake --version`, etc.). Show exact command outputs. +C) Propose the **smallest** change to either the fuzz target or the provided build script that resolves that error. \ No newline at end of file diff --git a/run_all_experiments.py b/run_all_experiments.py index 5568d4f55f..2ad1471e39 100755 --- a/run_all_experiments.py +++ b/run_all_experiments.py @@ -251,6 +251,11 @@ def parse_args() -> argparse.Namespace: action='store_true', default=False, help='Enables agent enhancement.') + parser.add_argument('-rag', + '--rag-classifier', + action='store_true', + default=True, + help='Enable the RAG-based build error classifier (default: on).') parser.add_argument('--custom-pipeline', type=str, default='') parser.add_argument('-mr', '--max-round', @@ -527,6 +532,9 @@ def main(): args = parse_args() _setup_logging(args.log_level, is_cloud=args.cloud_experiment_name != '') + # logger.info('[dbg] agent=%s rag_classifier=%s model=%s bench_yaml=%s bench_dir=%s', + # args.agent, args.rag_classifier, args.model, + # getattr(args, 'benchmark_yaml', ''), getattr(args, 'benchmarks_directory', '')) logger.info('Starting experiments on PR branch') # Capture time at start