fix conflict

Author: nanjiangwill

baselines/run_agent.py

@@ -52,7 +52,7 @@ def run_agent_for_repo(
     repo_name = repo_name.replace(".", "-")
 
     # Call the commit0 get-tests command to retrieve test files
-    test_files_str = get_tests(repo_name, stdout=False)
+    test_files_str = get_tests(repo_name, verbose=0)
     test_files = sorted(list(set([i.split(":")[0] for i in test_files_str])))
 
     repo_path = os.path.join(commit0_config.base_dir, repo_name)

commit0/cli.py

@@ -144,7 +144,13 @@ def build(
     num_workers: int = typer.Option(8, help="Number of workers"),
     commit0_dot_file_path: str = typer.Option(
         ".commit0.yaml",
-        help="Path to the commit0 dot file, where the setup config is stored",
+        help="Path to the commit0 dot file, where the setup config is stored"),
+    verbose: int = typer.Option(
+        1,
+        "--verbose",
+        "-v",
+        help="Set this to 2 for more logging information",
+        count=True,
     ),
 ) -> None:
     """Build Commit0 split you choose in Setup Stage."""
@@ -163,6 +169,7 @@ def build(
         commit0_config["dataset_split"],
         commit0_config["repo_split"],
         num_workers,
+        verbose,
     )
 
 
@@ -179,7 +186,7 @@ def get_tests(
 
     typer.echo(f"Getting tests for repository: {repo_name}")
 
-    commit0.harness.get_pytest_ids.main(repo_name, stdout=True)
+    commit0.harness.get_pytest_ids.main(repo_name, verbose=1)
 
 
 @commit0_app.command()
@@ -202,7 +209,13 @@ def test(
     ] = False,
     commit0_dot_file_path: str = typer.Option(
         ".commit0.yaml",
-        help="Path to the commit0 dot file, where the setup config is stored",
+        help="Path to the commit0 dot file, where the setup config is stored"),
+    verbose: int = typer.Option(
+        1,
+        "--verbose",
+        "-v",
+        help="Set this to 2 for more logging information",
+        count=True,
     ),
 ) -> None:
     """Run tests on a Commit0 repository."""
@@ -236,7 +249,7 @@ def test(
         backend,
         timeout,
         num_cpus,
-        stdout=True,
+        verbose,
     )
 
 

commit0/harness/build.py

@@ -19,6 +19,7 @@ def main(
     dataset_split: str,
     repo_split: str,
     num_workers: int,
+    verbose: int,
 ) -> None:
     dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
     specs = []
@@ -30,7 +31,11 @@ def main(
         specs.append(spec)
 
     client = docker.from_env()
-    build_repo_images(client, specs, num_workers)
+    build_repo_images(client, specs, num_workers, verbose)
+    for spec in specs:
+        image = client.images.get(spec.repo_image_key)
+        repository, tag = spec.repo_image_tag.split(":")
+        image.tag(repository, tag)
 
 
 __all__ = []

commit0/harness/docker_build.py

@@ -196,6 +196,7 @@ def build_repo_images(
     client: docker.DockerClient,
     dataset: list,
     max_workers: int = 4,
+    verbose: int = 1,
 ) -> tuple[list[str], list[str]]:
     """Builds the repo images required for the dataset if they do not already exist.
 
@@ -204,6 +205,7 @@ def build_repo_images(
         client (docker.DockerClient): Docker client to use for building the images
         dataset (list): List of test specs or dataset to build images for
         max_workers (int): Maximum number of workers to use for building images
+        verbose (int): Level of verbosity
 
     Return:
     ------

commit0/harness/docker_utils.py

@@ -10,7 +10,7 @@
 import traceback
 from pathlib import Path
 from io import BytesIO
-from typing import Optional, List, Union
+from typing import Optional, List
 
 import docker.errors
 from docker.models.containers import Container
@@ -126,7 +126,7 @@ def write_to_container(container: Container, data: str, dst: Path) -> None:
 def cleanup_container(
     client: docker.DockerClient,
     container: Container,
-    logger: Union[None, str, logging.Logger],
+    logger: logging.Logger,
 ) -> None:
     """Stop and remove a Docker container.
     Performs this forcefully if the container cannot be stopped with the python API.
@@ -135,51 +135,21 @@ def cleanup_container(
     ----
         client (docker.DockerClient): Docker client.
         container (docker.Container): Container to remove.
-        logger (Union[str, logging.Logger], optional): Logger instance or log level as string for logging container creation messages. Defaults to None.
+        logger (logging.Logger): Logger instance or log level as string for logging container creation messages.
 
     """
     if not container:
         return
 
     container_id = container.id
 
-    if not logger:
-        # if logger is None, print to stdout
-        def log_error(x: str) -> None:
-            print(x)
-
-        def log_info(x: str) -> None:
-            print(x)
-
-        raise_error = True
-    elif logger == "quiet":
-        # if logger is "quiet", don't print anything
-        def log_info(x: str) -> None:
-            return None
-
-        def log_error(x: str) -> None:
-            return None
-
-        raise_error = True
-    else:
-        assert isinstance(logger, logging.Logger)
-
-        # if logger is a logger object, use it
-        def log_error(x: str) -> None:
-            logger.info(x)
-
-        def log_info(x: str) -> None:
-            logger.info(x)
-
-        raise_error = False
-
     # Attempt to stop the container
     try:
         if container:
-            log_info(f"Attempting to stop container {container.name}...")
+            logger.info(f"Attempting to stop container {container.name}...")
             container.kill()
     except Exception as e:
-        log_error(
+        logger.error(
             f"Failed to stop container {container.name}: {e}. Trying to forcefully kill..."
         )
         try:
@@ -190,54 +160,109 @@ def log_info(x: str) -> None:
 
             # If container PID found, forcefully kill the container
             if pid > 0:
-                log_info(
+                logger.info(
                     f"Forcefully killing container {container.name} with PID {pid}..."
                 )
                 os.kill(pid, signal.SIGKILL)
             else:
-                log_error(f"PID for container {container.name}: {pid} - not killing.")
+                logger.error(
+                    f"PID for container {container.name}: {pid} - not killing."
+                )
         except Exception as e2:
-            if raise_error:
-                raise e2
-            log_error(
+            raise Exception(
                 f"Failed to forcefully kill container {container.name}: {e2}\n"
                 f"{traceback.format_exc()}"
             )
 
     # Attempt to remove the container
     try:
-        log_info(f"Attempting to remove container {container.name}...")
+        logger.info(f"Attempting to remove container {container.name}...")
         container.remove(force=True)
-        log_info(f"Container {container.name} removed.")
+        logger.info(f"Container {container.name} removed.")
     except Exception as e:
-        if raise_error:
-            raise e
-        log_error(
+        raise Exception(
             f"Failed to remove container {container.name}: {e}\n"
             f"{traceback.format_exc()}"
         )
 
 
+def image_exists_locally(
+    client: docker.DockerClient, image_name: str, tag: str, logger: logging.Logger
+) -> bool:
+    """Check if a Docker image exists locally.
+
+    Args:
+    ----
+    client (docker.DockerClient): Docker client instance.
+    image_name (str): The name of the Docker image.
+    tag (str, optional): Tag of the Docker image.
+    logger (logging.Logger): Logger instance.
+
+    Returns:
+    -------
+    bool: True if the image exists locally, False otherwise.
+
+    """
+    images = client.images.list(name=image_name)
+    for image in images:
+        if f"{image_name}:{tag}" in image.tags:
+            logger.info(f"Using {image_name}:{tag} found locally.")
+            return True
+    logger.info(f"{image_name}:{tag} cannot be found locally")
+    return False
+
+
+def pull_image_from_docker_hub(
+    client: docker.DockerClient, image_name: str, tag: str, logger: logging.Logger
+) -> None:
+    """Pull a Docker image from Docker Hub.
+
+    Args:
+    ----
+    client (docker.DockerClient): Docker client instance.
+    image_name (str): The name of the Docker image.
+    tag (str, optional): Tag of the Docker image.
+    logger (logging.Logger): Logger instance.
+
+    Returns:
+    -------
+    docker.models.images.Image: The pulled Docker image.
+
+    Raises:
+    ------
+    docker.errors.ImageNotFound: If the image is not found on Docker Hub.
+    docker.errors.APIError: If there's an issue with the Docker API during the pull.
+
+    """
+    try:
+        client.images.pull(image_name, tag=tag)
+        logger.info(f"Loaded {image_name}:{tag} from Docker Hub.")
+    except docker.errors.ImageNotFound:
+        raise Exception(f"Image {image_name}:{tag} not found on Docker Hub.")
+    except docker.errors.APIError as e:
+        raise Exception(f"Error pulling image: {e}")
+
+
 def create_container(
     client: docker.DockerClient,
     image_name: str,
-    container_name: Optional[str] = None,
+    container_name: str,
+    logger: logging.Logger,
     user: Optional[str] = None,
     command: Optional[str] = "tail -f /dev/null",
     nano_cpus: Optional[int] = None,
-    logger: Optional[Union[str, logging.Logger]] = None,
 ) -> Container:
     """Start a Docker container using the specified image.
 
     Args:
     ----
     client (docker.DockerClient): Docker client.
     image_name (str): The name of the Docker image.
-    container_name (str, optional): Name for the Docker container. Defaults to None.
+    container_name (str): Name for the Docker container.
+    logger (logging.Logger): Logger instance or log level as string for logging container creation messages.
     user (str, option): Log in as which user. Defaults to None.
     command (str, optional): Command to run in the container. Defaults to None.
     nano_cpus (int, optional): The number of CPUs for the container. Defaults to None.
-    logger (Union[str, logging.Logger], optional): Logger instance or log level as string for logging container creation messages. Defaults to None.
 
     Returns:
     -------
@@ -249,41 +274,13 @@ def create_container(
     Exception: For other general errors.
 
     """
-    try:
-        # Pull the image if it doesn't already exist
-        client.images.pull(image_name)
-    except docker.errors.APIError as e:
-        raise docker.errors.APIError(f"Error pulling image: {str(e)}")
-
-    if not logger:
-        # if logger is None, print to stdout
-        def log_error(x: str) -> None:
-            print(x)
-
-        def log_info(x: str) -> None:
-            print(x)
-
-    elif logger == "quiet":
-        # if logger is "quiet", don't print anything
-        def log_info(x: str) -> None:
-            return None
-
-        def log_error(x: str) -> None:
-            return None
-
-    else:
-        assert isinstance(logger, logging.Logger)
-
-        # if logger is a logger object, use it
-        def log_error(x: str) -> None:
-            logger.info(x)
-
-        def log_info(x: str) -> None:
-            logger.info(x)
+    image, tag = image_name.split(":")
+    if not image_exists_locally(client, image, tag, logger):
+        pull_image_from_docker_hub(client, image, tag, logger)
 
     container = None
     try:
-        log_info(f"Creating container for {image_name}...")
+        logger.info(f"Creating container for {image_name}...")
         container = client.containers.run(
             image=image_name,
             name=container_name,
@@ -292,12 +289,12 @@ def log_info(x: str) -> None:
             nano_cpus=nano_cpus,
             detach=True,
         )
-        log_info(f"Container for {image_name} created: {container.id}")
+        logger.info(f"Container for {image_name} created: {container.id}")
         return container
     except Exception as e:
         # If an error occurs, clean up the container and raise an exception
-        log_error(f"Error creating container for {image_name}: {e}")
-        log_info(traceback.format_exc())
+        logger.error(f"Error creating container for {image_name}: {e}")
+        logger.info(traceback.format_exc())
         assert container is not None
         cleanup_container(client, container, logger)
         raise

commit0/harness/evaluate.py

@@ -57,7 +57,7 @@ def main(
                     backend,
                     timeout,
                     num_cpus,
-                    stdout=False,
+                    verbose=0,
                 ): None
                 for repo, test_dir in pairs
             }
@@ -70,7 +70,7 @@ def main(
     for name in tqdm(log_dirs):
         report_file = os.path.join(name, "report.json")
         name = name.split("/")[2]
-        test_ids = get_tests(name, stdout=False)
+        test_ids = get_tests(name, verbose=0)
         if not os.path.exists(report_file):
             out.append(
                 {

commit0/harness/execution_context.py

@@ -99,7 +99,7 @@ def __init__(
         self.client = docker.from_env()
         self.container = create_container(
             client=self.client,
-            image_name=spec.repo_image_key,
+            image_name=spec.repo_image_tag,
             container_name=spec.get_container_name(),
             nano_cpus=num_cpus,
             logger=logger,