added splits for swebench

Author: wenting-zhao

commit0/harness/build.py

@@ -30,7 +30,7 @@ def main(
     for example in dataset:
         repo_name = example["repo"].split("/")[-1]
         if "swe" in dataset_name.lower():
-            if repo_split != "all" and example["instance_id"] != repo_split:
+            if repo_split != "all" and repo_split not in example["instance_id"]:
                 continue
         else:
             if repo_split != "all" and repo_name not in SPLIT[repo_split]:

commit0/harness/evaluate.py

@@ -36,15 +36,15 @@ def main(
         if repo_split == "all":
             repos = dataset["instance_id"]  # type: ignore
         else:
-            repos = [repo_split]
+            repos = [one for one in dataset["instance_id"] if repo_split in one]  # type: ignore
     else:
         repos = SPLIT[repo_split]
     triples = []
     log_dirs = []
     for example in dataset:
         repo_name = example["repo"].split("/")[-1]
         if "swe" in dataset_name.lower():
-            if repo_split != "all" and example["instance_id"] != repo_split:
+            if repo_split != "all" and repo_split not in example["instance_id"]:
                 continue
         else:
             if repo_split != "all" and repo_name not in SPLIT[repo_split]:

commit0/harness/save.py

@@ -31,7 +31,7 @@ def main(
     for example in dataset:
         repo_name = example["repo"].split("/")[-1]
         if "swe" in dataset_name.lower():
-            if repo_split != "all" and example["instance_id"] != repo_split:
+            if repo_split != "all" and repo_split not in example["instance_id"]:
                 continue
         else:
             if repo_split != "all" and repo_name not in SPLIT[repo_split]:

commit0/harness/setup.py

@@ -27,7 +27,7 @@ def main(
         repo_name = example["repo"].split("/")[-1]
         clone_url = f"https://github.com/{example['repo']}.git"
         if "swe" in dataset_name.lower():
-            if repo_split != "all" and example["instance_id"] != repo_split:
+            if repo_split != "all" and repo_split not in example["instance_id"]:
                 continue
             clone_dir = os.path.abspath(os.path.join(base_dir, example["instance_id"]))
             branch = example["base_commit"]