other integration

Author: wenting-zhao

commit0/cli.py

@@ -2,6 +2,7 @@
 from pathlib import Path
 from typing import Union, List
 from typing_extensions import Annotated
+from datasets import load_dataset
 import commit0.harness.run_pytest_ids
 import commit0.harness.get_pytest_ids
 import commit0.harness.build
@@ -118,7 +119,11 @@ def setup(
 ) -> None:
     """Commit0 clone a repo split."""
     check_commit0_path()
-    check_valid(repo_split, SPLIT)
+    if "swe" in dataset_name.lower():
+        SWE_SPLIT = load_dataset(dataset_name, split=dataset_split)["instance_id"]
+        check_valid(repo_split, SWE_SPLIT)
+    else:
+        check_valid(repo_split, SPLIT)
 
     base_dir = str(Path(base_dir).resolve())
 
@@ -168,7 +173,11 @@ def build(
     check_commit0_path()
 
     commit0_config = read_commit0_dot_file(commit0_dot_file_path)
-    check_valid(commit0_config["repo_split"], SPLIT)
+    if "swe" in commit0_config["dataset_name"].lower():
+        SWE_SPLIT = load_dataset(commit0_config["dataset_name"], split=commit0_config["dataset_split"])["instance_id"]
+        check_valid(commit0_config["repo_split"], SWE_SPLIT)
+    else:
+        check_valid(commit0_config["repo_split"], SPLIT)
 
     typer.echo(
         f"Building repository for split: {highlight(commit0_config['repo_split'], Colors.ORANGE)}"
@@ -199,7 +208,8 @@ def get_tests(
 ) -> None:
     """Get tests for a Commit0 repository."""
     check_commit0_path()
-    check_valid(repo_name, SPLIT_ALL)
+    SWE_SPLIT = load_dataset("princeton-nlp/SWE-bench_Verified", split="test")["instance_id"]
+    check_valid(repo_name, SPLIT_ALL+SWE_SPLIT)
 
     commit0.harness.get_pytest_ids.main(repo_name, verbose=1)
 
@@ -247,11 +257,14 @@ def test(
 ) -> None:
     """Run tests on a Commit0 repository."""
     check_commit0_path()
+    commit0_config = read_commit0_dot_file(commit0_dot_file_path)
     if repo_or_repo_path.endswith("/"):
         repo_or_repo_path = repo_or_repo_path[:-1]
-    check_valid(repo_or_repo_path.split("/")[-1], SPLIT_ALL)
-
-    commit0_config = read_commit0_dot_file(commit0_dot_file_path)
+    if "swe" in commit0_config["dataset_name"].lower():
+        SWE_SPLIT = load_dataset(commit0_config["dataset_name"], split=commit0_config["dataset_split"])["instance_id"]
+        check_valid(repo_or_repo_path.split("/")[-1], SWE_SPLIT)
+    else:
+        check_valid(repo_or_repo_path.split("/")[-1], SPLIT)
 
     if reference:
         branch = "reference"
@@ -316,7 +329,11 @@ def evaluate(
         branch = "reference"
 
     commit0_config = read_commit0_dot_file(commit0_dot_file_path)
-    check_valid(commit0_config["repo_split"], SPLIT)
+    if "swe" in commit0_config["dataset_name"].lower():
+        SWE_SPLIT = load_dataset(commit0_config["dataset_name"], split=commit0_config["dataset_split"])["instance_id"]
+        check_valid(commit0_config["repo_split"], SWE_SPLIT)
+    else:
+        check_valid(commit0_config["repo_split"], SPLIT)
 
     typer.echo(f"Evaluating repository split: {commit0_config['repo_split']}")
     typer.echo(f"Branch: {branch}")
@@ -391,7 +408,11 @@ def save(
     """Save Commit0 split you choose in Setup Stage to GitHub."""
     check_commit0_path()
     commit0_config = read_commit0_dot_file(commit0_dot_file_path)
-    check_valid(commit0_config["repo_split"], SPLIT)
+    if "swe" in commit0_config["dataset_name"].lower():
+        SWE_SPLIT = load_dataset(commit0_config["dataset_name"], split=commit0_config["dataset_split"])["instance_id"]
+        check_valid(commit0_config["repo_split"], SWE_SPLIT)
+    else:
+        check_valid(commit0_config["repo_split"], SPLIT)
 
     typer.echo(f"Saving repository split: {commit0_config['repo_split']}")
     typer.echo(f"Owner: {owner}")

commit0/harness/build.py

@@ -25,8 +25,12 @@ def main(
     specs = []
     for example in dataset:
         repo_name = example["repo"].split("/")[-1]
-        if repo_split != "all" and repo_name not in SPLIT[repo_split]:
-            continue
+        if "swe" in dataset_name.lower():
+            if repo_split != "all" and example["instance_id"] != repo_split:
+                continue
+        else:
+            if repo_split != "all" and repo_name not in SPLIT[repo_split]:
+                continue
         spec = make_spec(example)
         specs.append(spec)
 

commit0/harness/evaluate.py

@@ -37,8 +37,12 @@ def main(
     log_dirs = []
     for example in dataset:
         repo_name = example["repo"].split("/")[-1]
-        if repo_split != "all" and repo_name not in SPLIT[repo_split]:
-            continue
+        if "swe" in dataset_name.lower():
+            if repo_split != "all" and example["instance_id"] != repo_split:
+                continue
+        else:
+            if repo_split != "all" and repo_name not in SPLIT[repo_split]:
+                continue
         hashed_test_ids = get_hash_string(example["test"]["test_dir"])
         if branch is None:
             git_path = os.path.join(base_dir, repo_name)

commit0/harness/run_pytest_ids.py

@@ -51,7 +51,10 @@ def main(
     example = None
     repo_name = None
     for example in dataset:
-        repo_name = example["repo"].split("/")[-1]
+        if "swe" in dataset_name.lower():
+            repo_name = example["instance_id"]
+        else:
+            repo_name = example["repo"].split("/")[-1]
         if repo_or_repo_dir.endswith("/"):
             repo_or_repo_dir = repo_or_repo_dir[:-1]
         if repo_name in os.path.basename(repo_or_repo_dir):

commit0/harness/save.py

@@ -30,8 +30,12 @@ def main(
     dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
     for example in dataset:
         repo_name = example["repo"].split("/")[-1]
-        if repo_split != "all" and repo_name not in SPLIT[repo_split]:
-            continue
+        if "swe" in dataset_name.lower():
+            if repo_split != "all" and example["instance_id"] != repo_split:
+                continue
+        else:
+            if repo_split != "all" and repo_name not in SPLIT[repo_split]:
+                continue
         local_repo_path = f"{base_dir}/{repo_name}"
         github_repo_url = f"https://github.com/{owner}/{repo_name}.git"
         github_repo_url = github_repo_url.replace(

commit0/harness/setup.py

@@ -25,13 +25,15 @@ def main(
     dataset: Iterator[RepoInstance] = load_dataset(dataset_name, split=dataset_split)  # type: ignore
     for example in dataset:
         repo_name = example["repo"].split("/")[-1]
-        if repo_split != "all" and repo_name not in SPLIT[repo_split]:
-            continue
         clone_url = f"https://github.com/{example['repo']}.git"
         if "swe" in dataset_name.lower():
+            if repo_split != "all" and example["instance_id"] != repo_split:
+                continue
             clone_dir = os.path.abspath(os.path.join(base_dir, example["instance_id"]))
             branch = example["base_commit"]
         else:
+            if repo_split != "all" and repo_name not in SPLIT[repo_split]:
+                continue
             clone_dir = os.path.abspath(os.path.join(base_dir, repo_name))
             branch = dataset_name.split("/")[-1]
         repo = clone_repo(clone_url, clone_dir, branch, logger)

commit0/harness/spec.py

@@ -138,6 +138,8 @@ def make_repo_script_list(instance: RepoInstance, repo_directory: str) -> list[s
         setup_commands.append(cmd)
 
     if "install" in specs and specs["install"] is not None:
+        if specs["install"].startswith("python -m pip install"):
+            specs["install"] = specs["install"].replace("python -m ", "")
         if specs["install"].startswith("pip"):
             install = "uv " + specs["install"]
         else: