Repository navigation
Auto-quarantine (C7) #78
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Auto-quarantine (C7) | |
| # Runs daily at 04:15 UTC (1h after quarantine-sweep.yml at 02:23 UTC). | |
| # Detects tests that failed in an OSS Golden Path run then passed on a | |
| # re-run (= flaky). Opens a PR to add them to tests/quarantine.txt. | |
| # Tracking: vivekchand/clawmetry#3730 (C7) | |
| on: | |
| schedule: | |
| - cron: "15 4 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| dry_run: | |
| description: "Dry run (print but do not write quarantine.txt)" | |
| required: false | |
| default: "true" | |
| type: boolean | |
| lookback: | |
| description: "Number of recent workflow runs to scan" | |
| required: false | |
| default: "30" | |
| type: string | |
| concurrency: | |
| group: auto-quarantine | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| detect-and-quarantine: | |
| name: Detect flaky tests and quarantine within 24h | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| actions: read | |
| timeout-minutes: 10 | |
| steps: | |
| # persist-credentials is `true` on purpose, and is stated rather than | |
| # inherited: "Open quarantine PR" below runs `git push -u origin`, and the | |
| # only thing authenticating that push is the GITHUB_TOKEN this checkout | |
| # writes into .git/config. The GITHUB_TOKEN env on the neighbouring steps | |
| # authenticates `gh`, not `git`, so it would not cover the push if the | |
| # credential stopped persisting. Leaving the key off left that dependency | |
| # resting on an action default that is not ours to pin. Nothing in this job | |
| # uploads an artifact or caches the workspace, so the persisted credential | |
| # never leaves the runner. | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| persist-credentials: true | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.11" | |
| - name: Detect flaky tests | |
| id: detect | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| REPO_OWNER: vivekchand | |
| REPO_NAME: clawmetry | |
| # Dispatch inputs are untrusted text. Bind them to env vars and read | |
| # them as quoted shell variables so the runner never expands them as | |
| # code. Never interpolate an input into a `run:` block. | |
| DRY_RUN: ${{ inputs.dry_run }} | |
| LOOKBACK: ${{ inputs.lookback }} | |
| run: | | |
| ARGS=() | |
| if [ "$DRY_RUN" = "true" ] && [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| ARGS=(--dry-run) | |
| fi | |
| if [ -n "$LOOKBACK" ]; then | |
| ARGS+=(--lookback "$LOOKBACK") | |
| fi | |
| # GitHub Actions uses set -eo pipefail by default. Without set +e, | |
| # bash exits the run block immediately when auto_quarantine.py exits | |
| # with code 2 (new flaky tests found), so the echo line never runs | |
| # and steps.detect.outputs.exit_code is never set to '2'. | |
| # Fix: disable exit-on-error for the python call, capture the exit | |
| # code in SCRIPT_EXIT, then re-enable. | |
| set +e | |
| python3 scripts/auto_quarantine.py "${ARGS[@]}" | |
| SCRIPT_EXIT=$? | |
| set -e | |
| echo "exit_code=${SCRIPT_EXIT}" >> "$GITHUB_OUTPUT" | |
| continue-on-error: true | |
| - name: Open PR for newly quarantined tests | |
| if: steps.detect.outputs.exit_code == '2' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| git config user.email "actions@github.com" | |
| git config user.name "Auto-Quarantine Bot" | |
| BRANCH="auto-quarantine/$(date +%Y%m%d-%H%M%S)" | |
| git checkout -b "$BRANCH" | |
| git add tests/quarantine.txt | |
| git commit -m "test: auto-quarantine flaky E2E tests [C7] | |
| Flaky tests detected by auto_quarantine.py: failed in an OSS | |
| Golden Path run then passed on re-run of the same commit. Added to quarantine.txt | |
| so they are excluded from the main PR gate and swept daily. | |
| Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>" | |
| git push -u origin "$BRANCH" | |
| gh pr create \ | |
| --title "test: auto-quarantine flaky E2E tests (C7)" \ | |
| --body "$(cat <<'EOF' | |
| ## Summary | |
| Flaky tests detected by the daily auto-quarantine sweep (C7). | |
| These tests failed in a recent OSS Golden Path run then passed | |
| on a re-run of the same commit -- a clear flakiness signal. | |
| Added to \`tests/quarantine.txt\` so they are: | |
| - Excluded from the per-PR gate (no more PR blocks from flakiness) | |
| - Swept daily by \`quarantine-sweep.yml\` (alerts if they go from flaky to broken) | |
| ## Test plan | |
| - [ ] Review the quarantined test list in \`tests/quarantine.txt\` | |
| - [ ] Verify the root cause of each test's flakiness is tracked separately | |
| - [ ] Merge this PR to restore PR gate reliability | |
| Tracking: vivekchand/clawmetry#3730 (C7) | |
| EOF | |
| )" \ | |
| --label "e2e,testing" \ | |
| --base main \ | |
| --head "$BRANCH" | |
| - name: Summary | |
| if: always() | |
| env: | |
| DETECT_EXIT_CODE: ${{ steps.detect.outputs.exit_code }} | |
| run: | | |
| code="$DETECT_EXIT_CODE" | |
| case "$code" in | |
| 0) echo "No new flaky tests detected." ;; | |
| 2) echo "New flaky tests quarantined -- PR opened." ;; | |
| *) echo "Script error (exit $code) -- check logs above." ;; | |
| esac |