Deployed f99c131 with MkDocs version: 1.6.1

Author: celine-lee

analysis/index.html

@@ -401,7 +401,7 @@ <h2 id="leaderboard-lite">Leaderboard (lite)</h2>
 <tr>
 <th>Name</th>
 <th style="text-align: center;">Repos Resolved (/16)</th>
-<th style="text-align: center;">Total Tests Passed (/3628)</th>
+<th style="text-align: center;">% Tests Passed (Total: 3628)</th>
 <th style="text-align: center;">Test Duration (s)</th>
 <th style="text-align: center;">Date</th>
 <th>Analysis</th>
@@ -412,7 +412,7 @@ <h2 id="leaderboard-lite">Leaderboard (lite)</h2>
 <tr>
 <td>Reference (Gold)</td>
 <td style="text-align: center;">10</td>
-<td style="text-align: center;">3628</td>
+<td style="text-align: center;">100.00%</td>
 <td style="text-align: center;">21.00</td>
 <td style="text-align: center;">NA</td>
 <td><a href="/analysis_commit0_lite_reference">Analysis</a></td>
@@ -421,7 +421,7 @@ <h2 id="leaderboard-lite">Leaderboard (lite)</h2>
 <tr>
 <td>OpenHands (subset of <code>all</code>)</td>
 <td style="text-align: center;">2</td>
-<td style="text-align: center;">3217</td>
+<td style="text-align: center;">88.67%</td>
 <td style="text-align: center;">408.27</td>
 <td style="text-align: center;">11/25/2024</td>
 <td><a href="/analysis_openhands-commit0_openhands">Analysis</a></td>
@@ -430,7 +430,7 @@ <h2 id="leaderboard-lite">Leaderboard (lite)</h2>
 <tr>
 <td>Claude Sonnet 3.5 - Fill-in (subset of <code>all</code>)</td>
 <td style="text-align: center;">0</td>
-<td style="text-align: center;">694</td>
+<td style="text-align: center;">19.13%</td>
 <td style="text-align: center;">629.19</td>
 <td style="text-align: center;">09/25/2024</td>
 <td><a href="/analysis_commit0-all-plain_fillin">Analysis</a></td>
@@ -439,7 +439,7 @@ <h2 id="leaderboard-lite">Leaderboard (lite)</h2>
 <tr>
 <td>Claude Sonnet 3.5 - Fill-in + Unit Test Feedback</td>
 <td style="text-align: center;">0</td>
-<td style="text-align: center;">619</td>
+<td style="text-align: center;">17.06%</td>
 <td style="text-align: center;">552.79</td>
 <td style="text-align: center;">09/25/2024</td>
 <td><a href="/analysis_commit0-lite-with-test_fillin">Analysis</a></td>
@@ -448,7 +448,7 @@ <h2 id="leaderboard-lite">Leaderboard (lite)</h2>
 <tr>
 <td>Claude Sonnet 3.5 - Fill-in</td>
 <td style="text-align: center;">0</td>
-<td style="text-align: center;">381</td>
+<td style="text-align: center;">10.50%</td>
 <td style="text-align: center;">22.47</td>
 <td style="text-align: center;">09/25/2024</td>
 <td><a href="/analysis_commit0-lite-plain_fillin">Analysis</a></td>
@@ -457,7 +457,7 @@ <h2 id="leaderboard-lite">Leaderboard (lite)</h2>
 <tr>
 <td>Claude Sonnet 3.5 - Base</td>
 <td style="text-align: center;">0</td>
-<td style="text-align: center;">376</td>
+<td style="text-align: center;">10.36%</td>
 <td style="text-align: center;">16.83</td>
 <td style="text-align: center;">09/25/2024</td>
 <td><a href="/analysis_test-save-commit0_baseline">Analysis</a></td>
@@ -466,7 +466,7 @@ <h2 id="leaderboard-lite">Leaderboard (lite)</h2>
 <tr>
 <td>SWE-Agent (subset of <code>all</code>)</td>
 <td style="text-align: center;">0</td>
-<td style="text-align: center;">330</td>
+<td style="text-align: center;">9.10%</td>
 <td style="text-align: center;">62.35</td>
 <td style="text-align: center;">11/26/2024</td>
 <td><a href="/analysis_sweagent-commit0_sweagent">Analysis</a></td>
@@ -480,7 +480,7 @@ <h2 id="leaderboard-all">Leaderboard (all)</h2>
 <tr>
 <th>Name</th>
 <th style="text-align: center;">Repos Resolved (/56)</th>
-<th style="text-align: center;">Total Tests Passed (/140926)</th>
+<th style="text-align: center;">% Tests Passed (Total: 140926)</th>
 <th style="text-align: center;">Test Duration (s)</th>
 <th style="text-align: center;">Date</th>
 <th>Analysis</th>
@@ -491,7 +491,7 @@ <h2 id="leaderboard-all">Leaderboard (all)</h2>
 <tr>
 <td>Reference (Gold)</td>
 <td style="text-align: center;">19</td>
-<td style="text-align: center;">136766</td>
+<td style="text-align: center;">97.05%</td>
 <td style="text-align: center;">5467.81</td>
 <td style="text-align: center;">NA</td>
 <td><a href="/analysis_commit0_all_reference">Analysis</a></td>
@@ -500,7 +500,7 @@ <h2 id="leaderboard-all">Leaderboard (all)</h2>
 <tr>
 <td>OpenHands</td>
 <td style="text-align: center;">2</td>
-<td style="text-align: center;">3217</td>
+<td style="text-align: center;">2.28%</td>
 <td style="text-align: center;">408.27</td>
 <td style="text-align: center;">11/25/2024</td>
 <td><a href="/analysis_openhands-commit0_openhands">Analysis</a></td>
@@ -509,7 +509,7 @@ <h2 id="leaderboard-all">Leaderboard (all)</h2>
 <tr>
 <td>Claude Sonnet 3.5 - Fill-in</td>
 <td style="text-align: center;">0</td>
-<td style="text-align: center;">694</td>
+<td style="text-align: center;">0.49%</td>
 <td style="text-align: center;">629.19</td>
 <td style="text-align: center;">09/25/2024</td>
 <td><a href="/analysis_commit0-all-plain_fillin">Analysis</a></td>
@@ -518,7 +518,7 @@ <h2 id="leaderboard-all">Leaderboard (all)</h2>
 <tr>
 <td>SWE-Agent</td>
 <td style="text-align: center;">0</td>
-<td style="text-align: center;">330</td>
+<td style="text-align: center;">0.23%</td>
 <td style="text-align: center;">62.35</td>
 <td style="text-align: center;">11/26/2024</td>
 <td><a href="/analysis_sweagent-commit0_sweagent">Analysis</a></td>

render_submissions.py

@@ -164,7 +164,7 @@ def get_blank_repo_metrics(
 
 
 leaderboard_header = """\n\n## Leaderboard ({split})
-| Name | Repos Resolved (/{num_repos}) | Total Tests Passed (/{total_num_tests}) | Test Duration (s) | Date | Analysis | Github |
+| Name | Repos Resolved (/{num_repos}) | % Tests Passed (Total: {total_num_tests}) | Test Duration (s) | Date | Analysis | Github |
 |------|:-------------------------:|:--------------------:|:--------------------:|:----------:|----|----| """
 
 submission_table_header = """# Submission Name: **{display_name}** (split: {split})
@@ -334,7 +334,7 @@ def render_mds(overwrite_previous, subfolder="docs"):
             leaderboard[split].append((cum_tests_passed,
                 f"\n|{display_name}|"
                 f"{repos_resolved}|"
-                f"{cum_tests_passed}|"
+                f"{100.*cum_tests_passed/split_to_total_tests[split]:.2f}%|"
                 f"{total_duration:.2f}|"
                 f"{submission_date}|"
                 f"{analysis_link}|"
@@ -344,7 +344,7 @@ def render_mds(overwrite_previous, subfolder="docs"):
                 leaderboard["lite"].append((lite_cum_tests_passed,
                     f"\n|{display_name} (subset of `all`)|"
                     f"{lite_repos_resolved}|"
-                    f"{lite_cum_tests_passed}|"
+                    f"{100.*lite_cum_tests_passed/split_to_total_tests['lite']:.2f}%|"
                     f"{lite_total_duration:.2f}|"
                     f"{submission_date}|"
                     f"{analysis_link}|"