Skip to content

Commit 6b949c2

Browse files
celine-leeceline-lee
committed
linted
1 parent 17eaf38 commit 6b949c2

File tree

1 file changed

+112
-63
lines changed

1 file changed

+112
-63
lines changed

‎docs/render_submissions.py‎

Lines changed: 112 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,13 @@ def get_pytest_info(path_to_logs, repo_name, branch_name):
3939
}
4040
report_file_path = os.path.join(path_to_logs, pytest_hash, "report.json")
4141
if not os.path.exists(report_file_path):
42-
if os.path.exists(os.path.join(path_to_logs, pytest_hash, "test_output.txt")):
42+
if os.path.exists(
43+
os.path.join(path_to_logs, pytest_hash, "test_output.txt")
44+
):
4345
reason_for_failure = open(
4446
os.path.join(path_to_logs, pytest_hash, "test_output.txt")
4547
).read()
46-
else:
48+
else:
4749
reason_for_failure = "Unknown failure."
4850
pytest_info[testname]["failed_to_run"] = reason_for_failure
4951
return pytest_info
@@ -160,6 +162,7 @@ def get_blank_repo_metrics(
160162

161163
return blank_repo_metrics
162164

165+
163166
leaderboard_header = """\n\n## Leaderboard ({split})
164167
| Name | Repos Resolved (/{num_repos}) | Total Tests Passed (/{total_num_tests}) Test Duration (s) | Date | Analysis | Github |
165168
|------|:-------------------------:|:--------------------:|:--------------------:|:----------:|----|----| """
@@ -174,20 +177,26 @@ def get_blank_repo_metrics(
174177
|:---------|:-----:|
175178
"""
176179

180+
177181
def render_mds(overwrite_previous, subfolder="docs"):
178182
leaderboard = {}
179183

180-
split_to_total_tests = {"lite": 3628, "all": 140926} # hard-coded to skip running it later
184+
split_to_total_tests = {
185+
"lite": 3628,
186+
"all": 140926,
187+
} # hard-coded to skip running it later
181188
for split in tqdm.tqdm(["lite", "all"]):
182189
num_repos = len(SPLIT[split])
183190
# total_num_tests = 0
184191
# for repo_name in SPLIT[split]:
185192
# repo_tests = subprocess.run(['commit0', 'get-tests', repo_name], capture_output=True, text=True).stdout.strip()
186193
# total_num_tests += len(repo_tests.splitlines())
187-
leaderboard[
188-
split
189-
] = leaderboard_header.format(split=split, num_repos=num_repos, total_num_tests=split_to_total_tests[split])
190-
194+
leaderboard[split] = leaderboard_header.format(
195+
split=split,
196+
num_repos=num_repos,
197+
total_num_tests=split_to_total_tests[split],
198+
)
199+
191200
for org_path in tqdm.tqdm(glob.glob(os.path.join(analysis_files_path, "*"))):
192201
org_name = os.path.basename(org_path)
193202
if org_name in {"blank", "repos", "submission_repos"}:
@@ -204,58 +213,73 @@ def render_mds(overwrite_previous, subfolder="docs"):
204213
display_name = submission_info["display_name"]
205214
submission_date = submission_info["submission_date"]
206215
branch_name = submission_info["branch"]
207-
org_branch_filepath = os.path.join(subfolder, f"analysis_{org_name}_{branch_name}.md")
216+
org_branch_filepath = os.path.join(
217+
subfolder, f"analysis_{org_name}_{branch_name}.md"
218+
)
208219
write_submission = True
209-
if os.path.exists(org_branch_filepath) and not overwrite_previous: write_submission = False
220+
if os.path.exists(org_branch_filepath) and not overwrite_previous:
221+
write_submission = False
210222

211-
if write_submission: submission_page = submission_table_header.format(display_name=display_name, split=split)
223+
if write_submission:
224+
submission_page = submission_table_header.format(
225+
display_name=display_name, split=split
226+
)
212227

213228
for repo_name, repo_pytest_results in branch_metrics.items():
214-
if repo_name == "submission_info": continue
215-
if write_submission:
229+
if repo_name == "submission_info":
230+
continue
231+
if write_submission:
216232
submission_repo_page = f"# **{display_name}**: {repo_name}"
217-
org_branch_repo_filepath = os.path.join(subfolder, f"analysis_{org_name}_{branch_name}_{repo_name}.md")
218-
if isinstance(repo_pytest_results, str):
233+
org_branch_repo_filepath = os.path.join(
234+
subfolder, f"analysis_{org_name}_{branch_name}_{repo_name}.md"
235+
)
236+
if isinstance(repo_pytest_results, str):
219237
submission_repo_page = f"# **{display_name}**: {repo_name}\n\n## Failed to clone\n\n{repo_pytest_results}"
220-
org_branch_repo_filepath = os.path.join(subfolder, f"analysis_{org_name}_{branch_name}_{repo_name}.md")
221-
github_hyperlink = f"{project_page_link}/{repo_name}/tree/{branch_name}"
238+
org_branch_repo_filepath = os.path.join(
239+
subfolder, f"analysis_{org_name}_{branch_name}_{repo_name}.md"
240+
)
241+
github_hyperlink = (
242+
f"{project_page_link}/{repo_name}/tree/{branch_name}"
243+
)
222244
if branch_name == "reference":
223245
github_hyperlink = f"{project_page_link}/{repo_name}"
224-
submission_page = submission_table_header.format(display_name=display_name, split=split) + (
246+
submission_page = submission_table_header.format(
247+
display_name=display_name, split=split
248+
) + (
225249
f"| {repo_name} | No; Failed to clone. | - | - | "
226250
f"[Analysis](/{f'analysis_{org_name}_{branch_name}_{repo_name}'}) | "
227251
f"[Github]({github_hyperlink}) |"
228252
)
229-
back_button = (
230-
f"[back to {display_name} summary](/{f'analysis_{org_name}_{branch_name}'})\n\n"
231-
)
232-
with open(
233-
org_branch_repo_filepath, "w"
234-
) as wf:
253+
back_button = f"[back to {display_name} summary](/{f'analysis_{org_name}_{branch_name}'})\n\n"
254+
with open(org_branch_repo_filepath, "w") as wf:
235255
wf.write(back_button + submission_repo_page)
236256
continue
237257

238258
for pytest_group, pytest_info in repo_pytest_results.items():
239259
pytest_group = os.path.basename(pytest_group.strip("/"))
240-
patch_diff = (
241-
f"""\n\n## Patch diff\n```diff\n{pytest_info['patch_diff']}```"""
242-
)
260+
patch_diff = f"""\n\n## Patch diff\n```diff\n{pytest_info['patch_diff']}```"""
243261
if "failed_to_run" in pytest_info:
244262
resolved = False
245263
if write_submission:
246-
submission_repo_page += (f"\n## Failed to run pytests for test `{pytest_group}`\n"
247-
f"```\n{pytest_info['failed_to_run']}\n```")
264+
submission_repo_page += (
265+
f"\n## Failed to run pytests for test `{pytest_group}`\n"
266+
f"```\n{pytest_info['failed_to_run']}\n```"
267+
)
248268
pytest_details = "Pytest failed"
249269
duration = "Failed."
250270
else:
251271
resolved = ("failed" not in pytest_info["summary"]) or (
252272
pytest_info["summary"]["failed"] == 0
253273
)
254274
if write_submission:
255-
submission_repo_page += pytest_summary_table_header.format(pytest_group=pytest_group)
275+
submission_repo_page += pytest_summary_table_header.format(
276+
pytest_group=pytest_group
277+
)
256278
for category, count in pytest_info["summary"].items():
257279
if category not in {"duration"}:
258-
submission_repo_page += f"""| {category} | {count} |\n"""
280+
submission_repo_page += (
281+
f"""| {category} | {count} |\n"""
282+
)
259283
else:
260284
submission_repo_page += (
261285
f"""| {category} | {float(count):.2f}s |\n"""
@@ -275,21 +299,19 @@ def render_mds(overwrite_previous, subfolder="docs"):
275299
pytest_details = f"{pytest_info['summary']['passed']} / {pytest_info['summary']['collected']}"
276300
duration = f"{pytest_info['duration']:.2f}"
277301
break
278-
if write_submission:
279-
github_hyperlink = f"{project_page_link}/{repo_name}/tree/{branch_name}"
302+
if write_submission:
303+
github_hyperlink = (
304+
f"{project_page_link}/{repo_name}/tree/{branch_name}"
305+
)
280306
if branch_name == "reference":
281-
github_hyperlink = f"{project_page_link}/{repo_name}"
307+
github_hyperlink = f"{project_page_link}/{repo_name}"
282308
submission_page += (
283309
f"\n| {repo_name} | {'Yes' if resolved else 'No'} | {pytest_details} | "
284310
f"{duration} | [Analysis](/{f'analysis_{org_name}_{branch_name}_{repo_name}'}) | "
285311
f"[Github]({github_hyperlink}) |"
286312
)
287-
back_button = (
288-
f"[back to {display_name} summary](/{f'analysis_{org_name}_{branch_name}'})\n\n"
289-
)
290-
with open(
291-
org_branch_repo_filepath, "w"
292-
) as wf:
313+
back_button = f"[back to {display_name} summary](/{f'analysis_{org_name}_{branch_name}'})\n\n"
314+
with open(org_branch_repo_filepath, "w") as wf:
293315
wf.write(back_button + submission_repo_page + patch_diff)
294316
if write_submission:
295317
back_button = f"[back to all submissions](/{f'analysis'})\n\n"
@@ -307,7 +329,6 @@ def render_mds(overwrite_previous, subfolder="docs"):
307329
f"{github_link}|"
308330
)
309331

310-
311332
leaderboard_filepath = os.path.join(subfolder, "analysis.md")
312333
with open(leaderboard_filepath, "w") as wf:
313334
wf.write(leaderboard["lite"] + leaderboard["all"])
@@ -319,13 +340,19 @@ def get_args():
319340
"--do_setup", action="store_true", help="Run commit0 setup with specified split"
320341
)
321342
parser.add_argument(
322-
"--get_blank_details", action="store_true", help="Get difficulty metrics of blank repository"
343+
"--get_blank_details",
344+
action="store_true",
345+
help="Get difficulty metrics of blank repository",
323346
)
324347
parser.add_argument(
325-
"--get_reference_details", action="store_true", help="Get pytest results from reference"
348+
"--get_reference_details",
349+
action="store_true",
350+
help="Get pytest results from reference",
326351
)
327352
parser.add_argument(
328-
"--analyze_submissions", action="store_true", help="Get pytest results from submissions with split"
353+
"--analyze_submissions",
354+
action="store_true",
355+
help="Get pytest results from submissions with split",
329356
)
330357
parser.add_argument("--render_webpages", action="store_true")
331358
parser.add_argument("--split", type=str, help="all or lite")
@@ -334,7 +361,9 @@ def get_args():
334361
"--tokenizer_name", type=str, default="meta-llama/Meta-Llama-3.1-8B-Instruct"
335362
)
336363
parser.add_argument(
337-
"--overwrite_previous_eval", action="store_true", help="Overwrite cached pytest info"
364+
"--overwrite_previous_eval",
365+
action="store_true",
366+
help="Overwrite cached pytest info"
338367
# TODO add finer granularity so can specify which ones to overwrite
339368
)
340369

@@ -391,7 +420,9 @@ def main(args):
391420
commit0_dot_file_path = os.path.join(
392421
analysis_files_path, "repos", org_name, branch_name, ".commit0.yaml"
393422
)
394-
submission_repos_path = os.path.join(analysis_files_path, "repos", org_name, branch_name)
423+
submission_repos_path = os.path.join(
424+
analysis_files_path, "repos", org_name, branch_name
425+
)
395426
if args.do_setup:
396427
os.system(
397428
f"commit0 setup {args.split} --base-dir {submission_repos_path} "
@@ -400,22 +431,24 @@ def main(args):
400431
submission_metrics_output_file = os.path.join(
401432
analysis_files_path, org_name, f"{branch_name}.json"
402433
)
403-
submission_details = {"submission_info": {
404-
"org_name": org_name,
405-
"branch": branch_name,
406-
"display_name": "Reference (Gold)",
407-
"submission_date": "NA",
408-
"split": args.split,
409-
"project_page": "https://github.com/commit-0",
410-
}}
434+
submission_details = {
435+
"submission_info": {
436+
"org_name": org_name,
437+
"branch": branch_name,
438+
"display_name": "Reference (Gold)",
439+
"submission_date": "NA",
440+
"split": args.split,
441+
"project_page": "https://github.com/commit-0",
442+
}
443+
}
411444

412445
os.makedirs(os.path.join(analysis_files_path, org_name), exist_ok=True)
413446
need_re_eval = False
414447
for repo_log_path in glob.glob(f"{os.getcwd()}/logs/pytest/*"):
415448
if os.path.exists(os.path.join(repo_log_path, branch_name)):
416449
if args.overwrite_previous_eval:
417450
shutil.rmtree(os.path.join(repo_log_path, branch_name))
418-
else:
451+
else:
419452
need_re_eval = True
420453
if args.overwrite_previous_eval or need_re_eval:
421454
os.system(
@@ -431,27 +464,39 @@ def main(args):
431464
path_to_logs = f"{os.getcwd()}/logs/pytest/{repo_name}/{branch_name}"
432465
pytest_results = get_pytest_info(path_to_logs, repo_name, branch_name)
433466
submission_details[repo_name] = pytest_results
434-
json.dump(submission_details, open(submission_metrics_output_file, "w"), indent=4)
467+
json.dump(
468+
submission_details, open(submission_metrics_output_file, "w"), indent=4
469+
)
435470
print(f"Saved pytest info to {submission_metrics_output_file}")
436471

437472
if args.analyze_submissions:
438473
for submission in tqdm.tqdm(submission_dataset):
439474
submission_details = {"submission_info": submission}
440475
branch_name = submission["branch"]
441476
org_name = submission["org_name"]
442-
split = submission['split']
443-
if split != args.split: continue
477+
split = submission["split"]
478+
if split != args.split:
479+
continue
444480
submission_metrics_output_file = os.path.join(
445481
analysis_files_path, org_name, f"{branch_name}.json"
446482
)
447-
if os.path.exists(submission_metrics_output_file) and not args.overwrite_previous_eval:
483+
if (
484+
os.path.exists(submission_metrics_output_file)
485+
and not args.overwrite_previous_eval
486+
):
448487
continue
449-
submission_repos_path = os.path.join(analysis_files_path, "submission_repos", org_name, branch_name)
488+
submission_repos_path = os.path.join(
489+
analysis_files_path, "submission_repos", org_name, branch_name
490+
)
450491
if os.path.exists(submission_repos_path):
451492
shutil.rmtree(submission_repos_path)
452493
os.makedirs(os.path.join(analysis_files_path, org_name), exist_ok=True)
453494
commit0_dot_file_path = os.path.join(
454-
analysis_files_path, "submission_repos", org_name, branch_name, ".commit0.yaml"
495+
analysis_files_path,
496+
"submission_repos",
497+
org_name,
498+
branch_name,
499+
".commit0.yaml",
455500
)
456501
for repo_log_path in glob.glob(f"{os.getcwd()}/logs/pytest/*"):
457502
if os.path.exists(os.path.join(repo_log_path, branch_name)):
@@ -468,7 +513,8 @@ def main(args):
468513
clone_repo(clone_url, clone_dir, branch_name, logger)
469514
except Exception as e:
470515
submission_details[repo_name] = f"Error cloning: {e}"
471-
if os.path.exists(clone_dir): shutil.rmtree(clone_dir)
516+
if os.path.exists(clone_dir):
517+
shutil.rmtree(clone_dir)
472518
# after successfully setup, write the commit0 dot file
473519
write_commit0_dot_file(
474520
commit0_dot_file_path,
@@ -488,16 +534,19 @@ def main(args):
488534
repo_name = example["repo"].split("/")[-1]
489535
if split != "all" and repo_name not in SPLIT[split]:
490536
continue
491-
if repo_name in submission_details: # Failed to clone earlier, skip.
537+
if repo_name in submission_details: # Failed to clone earlier, skip.
492538
continue
493539
path_to_logs = f"{os.getcwd()}/logs/pytest/{repo_name}/{branch_name}"
494540
pytest_results = get_pytest_info(path_to_logs, repo_name, branch_name)
495541
submission_details[repo_name] = pytest_results
496-
json.dump(submission_details, open(submission_metrics_output_file, "w"), indent=4)
542+
json.dump(
543+
submission_details, open(submission_metrics_output_file, "w"), indent=4
544+
)
497545
print(f"Saved pytest info to {submission_metrics_output_file}")
498546

499547
if args.render_webpages:
500548
# Render only updated leaderboard and new submissions
501549
render_mds(args.overwrite_previous_eval)
502550

551+
503552
main(get_args())

0 commit comments

Comments
 (0)