Deployed bae65ce with MkDocs version: 1.6.1

Author: celine-lee

analysis/index.html

@@ -463,7 +463,7 @@ <h2 id="leaderboard-all">Leaderboard (all)</h2>
 <tbody>
 <tr>
 <td>Claude Sonnet 3.5 - Fill-in</td>
-<td style="text-align: center;">1</td>
+<td style="text-align: center;">0</td>
 <td style="text-align: center;">694</td>
 <td style="text-align: center;">629.19</td>
 <td style="text-align: center;">09/25/2024</td>
@@ -472,7 +472,7 @@ <h2 id="leaderboard-all">Leaderboard (all)</h2>
 </tr>
 <tr>
 <td>Reference (Gold)</td>
-<td style="text-align: center;">33</td>
+<td style="text-align: center;">19</td>
 <td style="text-align: center;">136766</td>
 <td style="text-align: center;">5467.81</td>
 <td style="text-align: center;">NA</td>

analysis_commit0-all-plain_fillin/index.html

@@ -346,7 +346,7 @@ <h1 id="submission-name-claude-sonnet-35-fill-in-split-all">Submission Name: <st
 <tr>
 <td>simpy</td>
 <td>No</td>
-<td style="text-align: center;">20 / 150</td>
+<td style="text-align: center;">20 / 140</td>
 <td style="text-align: center;">1.51</td>
 <td><a href="/analysis_commit0-all-plain_fillin_simpy">Analysis</a></td>
 <td><a href="https://github.com/commit0-all-plain/simpy/tree/fillin">Github</a></td>
@@ -409,7 +409,7 @@ <h1 id="submission-name-claude-sonnet-35-fill-in-split-all">Submission Name: <st
 </tr>
 <tr>
 <td>graphene</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">0 / 5</td>
 <td style="text-align: center;">0.97</td>
 <td><a href="/analysis_commit0-all-plain_fillin_graphene">Analysis</a></td>

analysis_commit0-lite-plain_fillin/index.html

@@ -330,7 +330,7 @@ <h1 id="submission-name-claude-sonnet-35-fill-in-split-lite">Submission Name: <s
 <tr>
 <td>simpy</td>
 <td>No</td>
-<td style="text-align: center;">20 / 150</td>
+<td style="text-align: center;">20 / 140</td>
 <td style="text-align: center;">1.43</td>
 <td><a href="/analysis_commit0-lite-plain_fillin_simpy">Analysis</a></td>
 <td><a href="https://github.com/commit0-lite-plain/simpy/tree/fillin">Github</a></td>

analysis_commit0-lite-with-test_fillin/index.html

@@ -330,7 +330,7 @@ <h1 id="submission-name-claude-sonnet-35-fill-in-unit-test-feedback-split-lite">
 <tr>
 <td>simpy</td>
 <td>No</td>
-<td style="text-align: center;">73 / 150</td>
+<td style="text-align: center;">73 / 140</td>
 <td style="text-align: center;">1.56</td>
 <td><a href="/analysis_commit0-lite-with-test_fillin_simpy">Analysis</a></td>
 <td><a href="https://github.com/commit0-lite-with-test/simpy/tree/fillin">Github</a></td>

analysis_commit0_all_reference/index.html

@@ -338,7 +338,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 <tr>
 <td>simpy</td>
 <td>Yes</td>
-<td style="text-align: center;">140 / 150</td>
+<td style="text-align: center;">140 / 140</td>
 <td style="text-align: center;">2.00</td>
 <td><a href="/analysis_commit0_all_reference_simpy">Analysis</a></td>
 <td><a href="https://github.com/commit-0/simpy">Github</a></td>
@@ -369,7 +369,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 </tr>
 <tr>
 <td>python-prompt-toolkit</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">150 / 151</td>
 <td style="text-align: center;">27.05</td>
 <td><a href="/analysis_commit0_all_reference_python-prompt-toolkit">Analysis</a></td>
@@ -385,7 +385,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 </tr>
 <tr>
 <td>pyjwt</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">259 / 262</td>
 <td style="text-align: center;">5.87</td>
 <td><a href="/analysis_commit0_all_reference_pyjwt">Analysis</a></td>
@@ -394,7 +394,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 <tr>
 <td>networkx</td>
 <td>No</td>
-<td style="text-align: center;">5436 / 10986</td>
+<td style="text-align: center;">5436 / 5493</td>
 <td style="text-align: center;">207.15</td>
 <td><a href="/analysis_commit0_all_reference_networkx">Analysis</a></td>
 <td><a href="https://github.com/commit-0/networkx">Github</a></td>
@@ -409,7 +409,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 </tr>
 <tr>
 <td>tlslite-ng</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">1653 / 1687</td>
 <td style="text-align: center;">10.68</td>
 <td><a href="/analysis_commit0_all_reference_tlslite-ng">Analysis</a></td>
@@ -466,7 +466,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 <tr>
 <td>web3.py</td>
 <td>No</td>
-<td style="text-align: center;">40795 / 42423</td>
+<td style="text-align: center;">40795 / 41771</td>
 <td style="text-align: center;">787.19</td>
 <td><a href="/analysis_commit0_all_reference_web3.py">Analysis</a></td>
 <td><a href="https://github.com/commit-0/web3.py">Github</a></td>
@@ -481,15 +481,15 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 </tr>
 <tr>
 <td>geopandas</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">1517 / 1590</td>
 <td style="text-align: center;">25.54</td>
 <td><a href="/analysis_commit0_all_reference_geopandas">Analysis</a></td>
 <td><a href="https://github.com/commit-0/geopandas">Github</a></td>
 </tr>
 <tr>
 <td>dulwich</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">1522 / 1587</td>
 <td style="text-align: center;">131.39</td>
 <td><a href="/analysis_commit0_all_reference_dulwich">Analysis</a></td>
@@ -529,7 +529,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 </tr>
 <tr>
 <td>requests</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">590 / 606</td>
 <td style="text-align: center;">78.79</td>
 <td><a href="/analysis_commit0_all_reference_requests">Analysis</a></td>
@@ -553,7 +553,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 </tr>
 <tr>
 <td>click</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">589 / 611</td>
 <td style="text-align: center;">1.46</td>
 <td><a href="/analysis_commit0_all_reference_click">Analysis</a></td>
@@ -569,7 +569,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 </tr>
 <tr>
 <td>statsmodels</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">17669 / 18189</td>
 <td style="text-align: center;">257.10</td>
 <td><a href="/analysis_commit0_all_reference_statsmodels">Analysis</a></td>
@@ -601,7 +601,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 </tr>
 <tr>
 <td>pydantic</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">5091 / 5141</td>
 <td style="text-align: center;">239.40</td>
 <td><a href="/analysis_commit0_all_reference_pydantic">Analysis</a></td>
@@ -617,15 +617,15 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 </tr>
 <tr>
 <td>pypdf</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">910 / 915</td>
 <td style="text-align: center;">74.22</td>
 <td><a href="/analysis_commit0_all_reference_pypdf">Analysis</a></td>
 <td><a href="https://github.com/commit-0/pypdf">Github</a></td>
 </tr>
 <tr>
 <td>attrs</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">1414 / 1419</td>
 <td style="text-align: center;">18.99</td>
 <td><a href="/analysis_commit0_all_reference_attrs">Analysis</a></td>
@@ -681,7 +681,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 </tr>
 <tr>
 <td>sphinx</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">2187 / 2211</td>
 <td style="text-align: center;">140.25</td>
 <td><a href="/analysis_commit0_all_reference_sphinx">Analysis</a></td>
@@ -729,7 +729,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 </tr>
 <tr>
 <td>filesystem_spec</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">698 / 792</td>
 <td style="text-align: center;">13.77</td>
 <td><a href="/analysis_commit0_all_reference_filesystem_spec">Analysis</a></td>
@@ -745,7 +745,7 @@ <h1 id="submission-name-reference-gold-split-all">Submission Name: <strong>Refer
 </tr>
 <tr>
 <td>sqlparse</td>
-<td>Yes</td>
+<td>No</td>
 <td style="text-align: center;">460 / 463</td>
 <td style="text-align: center;">1.48</td>
 <td><a href="/analysis_commit0_all_reference_sqlparse">Analysis</a></td>

analysis_commit0_lite_reference/index.html

@@ -330,7 +330,7 @@ <h1 id="submission-name-reference-gold-split-lite">Submission Name: <strong>Refe
 <tr>
 <td>simpy</td>
 <td>Yes</td>
-<td style="text-align: center;">140 / 150</td>
+<td style="text-align: center;">140 / 140</td>
 <td style="text-align: center;">2.00</td>
 <td><a href="/analysis_commit0_lite_reference_simpy">Analysis</a></td>
 <td><a href="https://github.com/commit-0/simpy">Github</a></td>

analysis_test-save-commit0_baseline/index.html

@@ -330,7 +330,7 @@ <h1 id="submission-name-claude-sonnet-35-base-split-lite">Submission Name: <stro
 <tr>
 <td>simpy</td>
 <td>No</td>
-<td style="text-align: center;">20 / 150</td>
+<td style="text-align: center;">20 / 140</td>
 <td style="text-align: center;">1.42</td>
 <td><a href="/analysis_test-save-commit0_baseline_simpy">Analysis</a></td>
 <td><a href="https://github.com/test-save-commit0/simpy/tree/baseline">Github</a></td>

render_submissions.py

@@ -268,9 +268,12 @@ def render_mds(overwrite_previous, subfolder="docs"):
                             pytest_details = "Pytest failed"
                             duration = "Failed."
                     else:
-                        resolved = ("failed" not in pytest_info["summary"]) or (
-                            pytest_info["summary"]["failed"] == 0
-                        )
+                        resolved = False
+                        if "passed" in pytest_info["summary"]:
+                            if "skipped" in pytest_info["summary"]:
+                                resolved = pytest_info["summary"]["passed"] + pytest_info["summary"]["skipped"] == pytest_info["summary"]["total"]
+                            else:
+                                resolved = pytest_info["summary"]["passed"] == pytest_info["summary"]["total"]
                         if write_submission:
                             submission_repo_page += pytest_summary_table_header.format(
                                 pytest_group=pytest_group
@@ -296,7 +299,7 @@ def render_mds(overwrite_previous, subfolder="docs"):
                         total_duration += pytest_info["duration"]
                         repos_resolved += int(resolved)
                         if write_submission:
-                            pytest_details = f"{pytest_info['summary']['passed']} / {pytest_info['summary']['collected']}"
+                            pytest_details = f"{pytest_info['summary']['passed']} / {pytest_info['summary']['total']}"
                             duration = f"{pytest_info['duration']:.2f}"
                     break
                 if write_submission: