Deployed 7e5b869 with MkDocs version: 1.6.1

Author: celine-lee

analysis/index.html

@@ -410,6 +410,15 @@ <h2 id="leaderboard-lite">Leaderboard (lite)</h2>
 </thead>
 <tbody>
 <tr>
+<td>Claude Sonnet 3.5 - Fill-in</td>
+<td style="text-align: center;">0</td>
+<td style="text-align: center;">448</td>
+<td style="text-align: center;">64.49</td>
+<td style="text-align: center;">09/25/2024</td>
+<td><a href="/analysis_commit0-all-plain_fillin">Analysis</a></td>
+<td><a href="https://github.com/commit0-all-plain">Github</a></td>
+</tr>
+<tr>
 <td>Claude Sonnet 3.5 - Base</td>
 <td style="text-align: center;">0</td>
 <td style="text-align: center;">376</td>
@@ -445,6 +454,33 @@ <h2 id="leaderboard-lite">Leaderboard (lite)</h2>
 <td><a href="/analysis_commit0_lite_reference">Analysis</a></td>
 <td><a href="https://github.com/commit-0">Github</a></td>
 </tr>
+<tr>
+<td>Reference (Gold)</td>
+<td style="text-align: center;">13</td>
+<td style="text-align: center;">10186</td>
+<td style="text-align: center;">166.64</td>
+<td style="text-align: center;">NA</td>
+<td><a href="/analysis_commit0_all_reference">Analysis</a></td>
+<td><a href="https://github.com/commit-0">Github</a></td>
+</tr>
+<tr>
+<td>OpenHands</td>
+<td style="text-align: center;">2</td>
+<td style="text-align: center;">1107</td>
+<td style="text-align: center;">116.76</td>
+<td style="text-align: center;">11/25/2024</td>
+<td><a href="/analysis_openhands-commit0_openhands">Analysis</a></td>
+<td><a href="https://github.com/openhands-commit0">Github</a></td>
+</tr>
+<tr>
+<td>SWE-Agent</td>
+<td style="text-align: center;">0</td>
+<td style="text-align: center;">284</td>
+<td style="text-align: center;">17.96</td>
+<td style="text-align: center;">11/26/2024</td>
+<td><a href="/analysis_sweagent-commit0_sweagent">Analysis</a></td>
+<td><a href="https://github.com/sweagent-commit0">Github</a></td>
+</tr>
 </tbody>
 </table>
 <h2 id="leaderboard-all">Leaderboard (all)</h2>

render_submissions.py

@@ -203,8 +203,11 @@ def render_mds(overwrite_previous, subfolder="docs"):
             continue
         for branch_path in glob.glob(os.path.join(org_path, "*.json")):
             cum_tests_passed = 0
+            lite_cum_tests_passed = 0
             repos_resolved = 0
+            lite_repos_resolved = 0
             total_duration = 0.0
+            lite_total_duration = 0.0
             branch_metrics = json.load(open(branch_path))
             submission_info = branch_metrics["submission_info"]
             split = submission_info["split"]
@@ -298,6 +301,10 @@ def render_mds(overwrite_previous, subfolder="docs"):
                         cum_tests_passed += pytest_info["summary"]["passed"]
                         total_duration += pytest_info["duration"]
                         repos_resolved += int(resolved)
+                        if repo_name in SPLIT["lite"]:
+                            lite_cum_tests_passed += pytest_info["summary"]["passed"]
+                            lite_total_duration += pytest_info["duration"]
+                            lite_repos_resolved += int(resolved)
                         if write_submission:
                             pytest_details = f"{pytest_info['summary']['passed']} / {pytest_info['summary']['total']}"
                             duration = f"{pytest_info['duration']:.2f}"
@@ -331,6 +338,16 @@ def render_mds(overwrite_previous, subfolder="docs"):
                 f"{analysis_link}|"
                 f"{github_link}|"
             )
+            if split == "all":
+                leaderboard["lite"] += (
+                    f"\n|{display_name}|"
+                    f"{lite_repos_resolved}|"
+                    f"{lite_cum_tests_passed}|"
+                    f"{lite_total_duration:.2f}|"
+                    f"{submission_date}|"
+                    f"{analysis_link}|"
+                    f"{github_link}|"
+                )
 
     leaderboard_filepath = os.path.join(subfolder, "analysis.md")
     with open(leaderboard_filepath, "w") as wf: