Updates to LUCIA tool

nimrod/__main__.py

@@ -25,7 +25,23 @@
 from nimrod.input_parsing.input_parser import CsvInputParser, JsonInputParser
 
 
-def get_test_suite_generators(config: Dict[str, Any]) -> List[TestSuiteGenerator]:
+def get_llm_test_suite_generators(config: Dict[str, Any]) -> List[TestSuiteGenerator]:
+  """
+  Creates test suite generators for all available LLM models configured in api_params.
+  Each model gets its own generator instance to enable parallel processing and comparison.
+  """
+  generators: List[TestSuiteGenerator] = list()
+  api_params = config.get('api_params', {})
+
+  for model_key, model_config in api_params.items():
+    # Create a generator for each configured model
+    generator = OllamaTestSuiteGenerator(Java(), model_key, model_config)
+    generators.append(generator)
+
+  return generators
+
+
+def get_test_suite_generators(config: Dict[str, str]) -> List[TestSuiteGenerator]:
   config_generators = config.get(
       'test_suite_generators', ['randoop', 'randoop-modified', 'evosuite', 'evosuite-differential', 'ollama', 'project'])
   generators: List[TestSuiteGenerator] = list()
@@ -40,20 +56,15 @@ def get_test_suite_generators(config: Dict[str, Any]) -> List[TestSuiteGenerator
   if 'evosuite-differential' in config_generators:
     generators.append(EvosuiteDifferentialTestSuiteGenerator(Java()))
   if 'ollama' in config_generators:
-    # Create one generator instance for each configured model
-    api_params = config.get('api_params', {})
-    if api_params:
-      for model_key, model_config in api_params.items():
-        generators.append(OllamaTestSuiteGenerator(Java(), model_key, model_config))
-    else:
-      generators.append(OllamaTestSuiteGenerator(Java()))
+    # Create one generator for each configured model
+    generators.extend(get_llm_test_suite_generators(config))
   if 'project' in config_generators:
     generators.append(ProjectTestSuiteGenerator(Java()))
 
   return generators
 
 
-def get_output_generators(config: Dict[str, Any]) -> List[OutputGenerator]:
+def get_output_generators(config: Dict[str, str]) -> List[OutputGenerator]:
   config_generators = config.get(
       'output_generators', ['behavior_changes', 'semantic_conflicts', 'test_suites'])
   generators: List[OutputGenerator] = list()
@@ -69,7 +80,7 @@ def get_output_generators(config: Dict[str, Any]) -> List[OutputGenerator]:
   return generators
 
 
-def parse_scenarios_from_input(config: Dict[str, Any]) -> List[MergeScenarioUnderAnalysis]:
+def parse_scenarios_from_input(config: Dict[str, str]) -> List[MergeScenarioUnderAnalysis]:
     json_input = config.get('input_path', "")
     csv_input_path = config.get('path_hash_csv', "")
 

nimrod/test_suite_generation/generators/llm_output_processor.py

@@ -5,7 +5,7 @@
 
 class OutputSanitizationRule(ABC):
     """Abstract base class for output sanitization rules."""
-    
+
     @abstractmethod
     def apply(self, output: str) -> str:
         """Apply the sanitization rule to the output."""
@@ -14,32 +14,32 @@ def apply(self, output: str) -> str:
 
 class RemoveThinkTagsRule(OutputSanitizationRule):
     """Removes content between <think> tags."""
-    
+
     def apply(self, output: str) -> str:
         return re.sub(r'<think>.*?</think>', '', output, flags=re.DOTALL)
 
 
 class ExtractCodeBlocksRule(OutputSanitizationRule):
     """Extracts content from code blocks (``` markers)."""
-    
+
     def apply(self, output: str) -> str:
         matches = re.findall(r'```(?:\w+)?\n?(.*?)```', output, flags=re.DOTALL)
         return '\n'.join(matches).strip()
 
 
 class RemoveNumberedLinesRule(OutputSanitizationRule):
     """Removes lines starting with 'number. <text>' pattern."""
-    
+
     def apply(self, output: str) -> str:
         return re.sub(r"^\d+\.\s.*$", "", output, flags=re.MULTILINE)
 
 
 class ExtractFromAnnotationsRule(OutputSanitizationRule):
     """Keeps only content starting from the first annotation marker."""
-    
+
     def __init__(self, markers: Optional[List[str]] = None):
         self.markers = markers or ["@Before", "@BeforeClass", "@Test"]
-    
+
     def apply(self, output: str) -> str:
         index = min(
             (output.find(marker) for marker in self.markers if marker in output), 
@@ -55,11 +55,11 @@ class LLMOutputProcessor:
     This class provides a flexible framework for cleaning LLM outputs
     by applying a series of sanitization rules in sequence.
     """
-    
+
     def __init__(self) -> None:
         self._rules: List[OutputSanitizationRule] = []
         self._load_default_rules()
-    
+
     def _load_default_rules(self) -> None:
         """Load the default set of sanitization rules."""
         self._rules = [
@@ -68,19 +68,19 @@ def _load_default_rules(self) -> None:
             RemoveNumberedLinesRule(),
             ExtractFromAnnotationsRule()
         ]
-    
+
     def add_rule(self, rule: OutputSanitizationRule) -> None:
         """Add a custom sanitization rule."""
         self._rules.append(rule)
-    
+
     def remove_rule(self, rule_type: type) -> None:
         """Remove all rules of the specified type."""
         self._rules = [rule for rule in self._rules if not isinstance(rule, rule_type)]
-    
+
     def clear_rules(self) -> None:
         """Remove all sanitization rules."""
         self._rules.clear()
-    
+
     def process(self, output: str) -> str:
         """
         Process the LLM output by applying all sanitization rules in sequence.
@@ -92,17 +92,17 @@ def process(self, output: str) -> str:
             The processed and sanitized output
         """
         processed_output = output
-        
+
         for rule in self._rules:
             try:
                 processed_output = rule.apply(processed_output)
             except Exception as e:
                 # Log the error but continue processing with other rules
                 import logging
                 logging.warning(f"Error applying rule {rule.__class__.__name__}: {e}")
-        
+
         return processed_output
-    
+
     def get_active_rules(self) -> List[str]:
         """Get the names of currently active rules."""
-        return [rule.__class__.__name__ for rule in self._rules]
+        return [rule.__class__.__name__ for rule in self._rules]

nimrod/test_suite_generation/generators/ollama_test_suite_generator.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import os
-import requests  # type: ignore
+import requests # type: ignore
 from typing import List, Dict, Union, Any, Optional
 import re
 
@@ -79,7 +79,7 @@ def generate_output(self, messages: List[Dict[str, str]]) -> Dict[str, Any]:
         except Exception as e:
             logging.error(f"Error generating output: {e}")
             return {"error": "Output generation error", "total_duration": self.timeout_seconds * 1_000_000_000}
-        
+
 
 class OllamaTestSuiteGenerator(TestSuiteGenerator):
 
@@ -113,7 +113,7 @@ def generate_messages_list(self, method_info: Dict[str, str], full_class_name: s
         api.set_branch(branch)  # Set the branch
         class_name = full_class_name.split('.')[-1]
         method_name = method_info.get("method_name", "")
-        
+
         logging.debug(f"Generating messages for {class_name}.{method_name} using template: {self.prompt_template}")
 
         # Generates messages using the PromptManager
@@ -123,15 +123,15 @@ def generate_messages_list(self, method_info: Dict[str, str], full_class_name: s
             branch=branch,
             template_name=self.prompt_template
         )
-        
+
         # Saves generated messages
         self.prompt_manager.save_generated_messages(
             messages_dict=messages_dict,
             output_path=output_path,
             class_name=class_name,
             method_name=method_name
         )
-        
+
         logging.info(f"Generated {len(messages_dict)} prompt variations for {class_name}.{method_name}")
 
         return messages_dict
@@ -162,7 +162,7 @@ def _ensure_api_initialized(self) -> Api:
             model=model_params.get("model", "codellama:70b")
         )
         return self.api
-
+    
     def _get_api_instance(self) -> Api:
         """Returns a valid API instance, initializing it if necessary."""
         return self._ensure_api_initialized()
@@ -202,10 +202,10 @@ def save_output(self, test_template: str, output: str, dir: str, output_file_nam
         """Saves the output generated by the model to a file, replacing #TEST_METHODS# in the template."""
         # Process and sanitize the LLM output using the configured processor
         processed_output = self.output_processor.process(output)
-        
+
         # Replace the placeholder in the template with the processed content
         filled_template = test_template.replace("#TEST_METHODS#", processed_output)
-        
+
         # Save to file
         llm_outputs_dir = os.path.join(dir, "llm_outputs")
         output_file_path = os.path.join(llm_outputs_dir, f"{output_file_name}.txt")
@@ -247,12 +247,22 @@ def extract_class_info(self, source_code_path: str, full_method_name: str, full_
                             name: (identifier) @constructor_name) @constructor_declaration
                         (method_declaration
                             name: (identifier) @method_name) @method_def
+                        (class_declaration) @nested_class
                         (#eq? @method_name "{method_name}")
                         (#eq? @constructor_name "{class_name}")
                         ]
                     )
                     (#eq? @class_name "{class_name}")
                 )
+
+                ; Searches for methods in any class (including nested ones)
+                (method_declaration
+                    name: (identifier) @any_method_name
+                    (#eq? @any_method_name "{method_name}")
+                ) @any_method_def
+
+                ; Searches for fields in any class (including nested ones)
+                (field_declaration) @any_field_declaration
             """
             query = JAVA_LANGUAGE.query(query_text)
             cursor = QueryCursor(query)
@@ -279,6 +289,34 @@ def extract_class_info(self, source_code_path: str, full_method_name: str, full_
                     class_constructors.append(captured_text)
                 elif capture_name == "method_def":
                     class_method = captured_text
+                elif capture_name == "any_method_def" and not class_method:
+                    class_method = captured_text
+                elif capture_name == "any_field_declaration" and method_name in captured_text and not class_method:
+                    class_method = captured_text
+
+            if not class_method:
+                logging.debug(f"Method/field '{method_name}' not found with main query, trying broader field search")
+
+                # More specific query for fields with exact name (including in nested classes)
+                field_query_text = f"""
+                    (field_declaration
+                        declarator: (variable_declarator
+                            name: (identifier) @field_name
+                        )
+                        (#eq? @field_name "{method_name}")
+                    ) @target_field_declaration
+                """
+
+                field_query = JAVA_LANGUAGE.query(field_query_text)
+                field_cursor = QueryCursor(field_query)
+                field_captures_dict = field_cursor.captures(tree.root_node)
+
+                for capture_name, nodes in field_captures_dict.items():
+                    for node in nodes:
+                        if capture_name == "target_field_declaration":
+                            class_method = self.extract_snippet(source_code, node.start_byte, node.end_byte)
+                            logging.debug(f"Found field '{method_name}' as target: {class_method[:100]}...")
+                            break
 
             return class_fields, class_constructors, class_method
 

nimrod/tests/env-config.json

@@ -20,4 +20,4 @@
       "timeout_seconds": 300
     }
   }
-}
+}

nimrod/tests/utils.py

@@ -1,7 +1,10 @@
 import logging
+import logging.handlers
 import os
 import json
 import shutil
+from datetime import datetime
+from pathlib import Path
 from typing import Dict, Any
 
 PATH = os.path.dirname(os.path.abspath(__file__))
@@ -71,29 +74,57 @@ def calculator_sum_aor_1():
 
 
 def setup_logging():
-    config = get_config()
-    config_level = config.get('logger_level', 'INFO').upper()
-    level = logging._nameToLevel.get(config_level, logging.INFO)
+    try:
+        config = get_config()
+        config_level = config.get('logger_level', 'INFO').upper()
+    except (FileNotFoundError, json.JSONDecodeError, KeyError):
+        config_level = 'INFO'
+
+    level = getattr(logging, config_level, logging.INFO)
 
     logger = logging.getLogger()
     logger.setLevel(level)
 
     if logger.hasHandlers():
         logger.handlers.clear()
 
-    formatter = logging.Formatter(
-        '[%(asctime)s] %(levelname)s %(filename)s:%(lineno)d: %(message)s',
+    log_dir = Path("logs")
+    log_dir.mkdir(exist_ok=True)
+
+    detailed_formatter = logging.Formatter(
+        '[%(asctime)s] %(levelname)-8s [%(name)s:%(lineno)d] %(funcName)s() - %(message)s',
         datefmt='%Y-%m-%d %H:%M:%S'
     )
+
+    console_formatter = logging.Formatter(
+        '[%(asctime)s] %(levelname)-8s - %(message)s',
+        datefmt='%H:%M:%S'
+    )
 
-    file_handler = logging.FileHandler('logfile.log', mode='a')
-    file_handler.setFormatter(formatter)
-
-    stream_handler = logging.StreamHandler()
-    stream_handler.setFormatter(formatter)
-
-    logger.addHandler(file_handler)
-    logger.addHandler(stream_handler)
+    try:
+        main_log_file = log_dir / "smat_nimrod.log"
+        main_handler = logging.handlers.RotatingFileHandler(
+            main_log_file,
+            maxBytes=50 * 1024 * 1024,  # 50MB
+            backupCount=5,
+            encoding='utf-8'
+        )
+        main_handler.setLevel(logging.DEBUG)
+        main_handler.setFormatter(detailed_formatter)
+        logger.addHandler(main_handler)
+    except (OSError, PermissionError):
+        fallback_log = log_dir / f"smat_nimrod_{datetime.now().strftime('%Y%m%d')}.log"
+        fallback_handler = logging.FileHandler(fallback_log, mode='a', encoding='utf-8')
+        fallback_handler.setLevel(logging.DEBUG)
+        fallback_handler.setFormatter(detailed_formatter)
+        logger.addHandler(fallback_handler)
+
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(level)
+    console_handler.setFormatter(console_formatter)
+    logger.addHandler(console_handler)
+
+    logger.info("SMAT Logging initialized - Level: %s", config_level)
 
 def get_base_output_path() -> str:
     current_dir = os.getcwd()

setup.py

@@ -25,7 +25,7 @@ def readme():
         'argparse==1.4.0',
         'beautifulsoup4==4.6.0',
         'pygithub==2.5.0',
-        'gitpython==3.1.41'
+        'gitpython==3.1.50'
     ],
     test_suite='nose.collector',
     tests_require=[