Merge pull request #42 from commit-0/handle-log-level

Handle log level

Author: wenting-zhao

baselines/run_agent.py

@@ -52,7 +52,7 @@ def run_agent_for_repo(
     repo_name = repo_name.replace(".", "-")
 
     # Call the commit0 get-tests command to retrieve test files
-    test_files_str = get_tests(repo_name, stdout=False)
+    test_files_str = get_tests(repo_name, verbose=0)
     test_files = sorted(list(set([i.split(":")[0] for i in test_files_str])))
 
     repo_path = os.path.join(commit0_config.base_dir, repo_name)

commit0/cli.py

@@ -77,6 +77,13 @@ def build(
     ),
     dataset_split: str = typer.Option("test", help="Split of the Huggingface dataset"),
     num_workers: int = typer.Option(8, help="Number of workers"),
+    verbose: int = typer.Option(
+        1,
+        "--verbose",
+        "-v",
+        help="Set this to 2 for more logging information",
+        count=True,
+    ),
 ) -> None:
     """Commit0 build a repository."""
     check_valid(repo_split, SPLIT)
@@ -91,6 +98,7 @@ def build(
         dataset_split,
         repo_split,
         num_workers,
+        verbose,
     )
 
 
@@ -106,7 +114,7 @@ def get_tests(
 
     typer.echo(f"Getting tests for repository: {repo_name}")
 
-    commit0.harness.get_pytest_ids.main(repo_name, stdout=True)
+    commit0.harness.get_pytest_ids.main(repo_name, verbose=1)
 
 
 @app.command()
@@ -132,6 +140,13 @@ def test(
     reference: Annotated[
         bool, typer.Option("--reference", help="Test the reference commit.")
     ] = False,
+    verbose: int = typer.Option(
+        1,
+        "--verbose",
+        "-v",
+        help="Set this to 2 for more logging information",
+        count=True,
+    ),
 ) -> None:
     """Run tests on a Commit0 repository."""
     if repo_or_repo_path.endswith("/"):
@@ -160,7 +175,7 @@ def test(
         backend,
         timeout,
         num_cpus,
-        stdout=True,
+        verbose,
     )
 
 

commit0/harness/build.py

@@ -19,6 +19,7 @@ def main(
     dataset_split: str,
     repo_split: str,
     num_workers: int,
+    verbose: int,
 ) -> None:
     dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
     specs = []
@@ -30,7 +31,7 @@ def main(
         specs.append(spec)
 
     client = docker.from_env()
-    build_repo_images(client, specs, num_workers)
+    build_repo_images(client, specs, num_workers, verbose)
     for spec in specs:
         image = client.images.get(spec.repo_image_key)
         repository, tag = spec.repo_image_tag.split(":")

commit0/harness/docker_build.py

@@ -196,6 +196,7 @@ def build_repo_images(
     client: docker.DockerClient,
     dataset: list,
     max_workers: int = 4,
+    verbose: int = 1,
 ) -> tuple[list[str], list[str]]:
     """Builds the repo images required for the dataset if they do not already exist.
 
@@ -204,6 +205,7 @@ def build_repo_images(
         client (docker.DockerClient): Docker client to use for building the images
         dataset (list): List of test specs or dataset to build images for
         max_workers (int): Maximum number of workers to use for building images
+        verbose (int): Level of verbosity
 
     Return:
     ------

commit0/harness/evaluate.py

@@ -57,7 +57,7 @@ def main(
                     backend,
                     timeout,
                     num_cpus,
-                    stdout=False,
+                    verbose=0,
                 ): None
                 for repo, test_dir in pairs
             }
@@ -70,7 +70,7 @@ def main(
     for name in tqdm(log_dirs):
         report_file = os.path.join(name, "report.json")
         name = name.split("/")[2]
-        test_ids = get_tests(name, stdout=False)
+        test_ids = get_tests(name, verbose=0)
         if not os.path.exists(report_file):
             out.append(
                 {

commit0/harness/get_pytest_ids.py

@@ -2,7 +2,7 @@
 from typing import List
 
 
-def main(repo: str, stdout: bool) -> List[str]:
+def main(repo: str, verbose: int) -> List[str]:
     repo = repo.lower()
     repo = repo.replace(".", "-")
     out = ""
@@ -13,7 +13,7 @@ def main(repo: str, stdout: bool) -> List[str]:
                 if file:
                     content = file.read().decode("utf-8")
                     out += content
-                    if stdout:
+                    if verbose:
                         print(content)
     out = out.split("\n")
     return out

commit0/harness/run_pytest_ids.py

@@ -37,7 +37,7 @@ def main(
     backend: str,
     timeout: int,
     num_cpus: int,
-    stdout: bool,
+    verbose: int,
 ) -> None:
     """Runs the pytests for repos in a dataset.
 
@@ -64,15 +64,17 @@ def main(
     log_dir = RUN_PYTEST_LOG_DIR / repo_name / branch / hashed_test_ids
     log_dir.mkdir(parents=True, exist_ok=True)
     log_file = log_dir / "run_pytest.log"
-    logger = setup_logger(repo_name, log_file)
+    logger = setup_logger(repo_name, log_file, verbose=verbose)
 
     try:
         local_repo = git.Repo(repo_or_repo_dir)
+        logger.info(f"Loaded a git repo from {repo_or_repo_dir}")
     except git.exc.NoSuchPathError:  # type: ignore
         repo_dir = os.path.join(base_dir, repo_name)
         logger.error(f"{repo_or_repo_dir} is not a git dir, trying {repo_dir} again")
         try:
             local_repo = git.Repo(repo_dir)
+            logger.info(f"Retried succeeded. Loaded a git repo from {repo_dir}")
         except git.exc.NoSuchPathError:  # type: ignore
             raise Exception(
                 f"{repo_dir} and {repo_or_repo_dir} are not git directories.\nUsage: commit0 test {{repo_dir}} {{branch}} {{test_ids}}"
@@ -133,6 +135,9 @@ def main(
                     logger,
                 )
         close_logger(logger)
+        if verbose > 0:
+            test_output = Path(log_dir / "test_output.txt")
+            print(test_output.read_text())
         pytest_exit_code = Path(log_dir / "pytest_exit_code.txt").read_text().strip()
         sys.exit(int(pytest_exit_code))
     except EvaluationError as e:

commit0/harness/utils.py

@@ -27,18 +27,22 @@ def __str__(self):
         )
 
 
-def setup_logger(repo: str, log_file: Path, mode: str = "w") -> logging.Logger:
+def setup_logger(
+    repo: str, log_file: Path, mode: str = "w", verbose: int = 1
+) -> logging.Logger:
     """Used for logging the build process of images and running containers.
     It writes logs to the log file.
     """
     log_file.parent.mkdir(parents=True, exist_ok=True)
     logger = logging.getLogger(f"{repo}.{log_file.name}")
     handler = logging.FileHandler(log_file, mode=mode)
-    stdout_handler = logging.StreamHandler(sys.stdout)
-    logger.addHandler(stdout_handler)
     formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
     handler.setFormatter(formatter)
     logger.addHandler(handler)
+    if verbose == 2:
+        stdout_handler = logging.StreamHandler(sys.stdout)
+        stdout_handler.setFormatter(formatter)
+        logger.addHandler(stdout_handler)
     logger.setLevel(logging.INFO)
     logger.propagate = False
     setattr(logger, "log_file", log_file)