Live Playwright driver feeding the computer-use recorder (closes #4) #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Example: gate a pull request on agent reliability and post the result as a sticky PR comment. | |
| # Copy this into your own repo and point --agent / the baseline at your agent. | |
| name: volo-pr-reliability | |
| on: | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| pull-requests: write # needed to post the reliability comment | |
| jobs: | |
| reliability: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: "0.8.x" | |
| - run: uv python install 3.12 | |
| - run: uv sync --all-packages | |
| # Record a baseline (in your repo, commit a baseline recording instead and skip this). | |
| - name: record baseline | |
| run: | | |
| mkdir -p .volo/recordings .volo/reports | |
| uv run volo record examples.calc_agent:run \ | |
| --input '{"a":2,"b":3,"c":4}' \ | |
| --out ./.volo/recordings/calc.json --data-dir ./.volo | |
| # Run the suite. `--summary-md` writes the PR-comment body; also feeds the job summary. | |
| # We don't fail the step here so the comment is always posted; the gate step below fails. | |
| - name: volo reliability suite | |
| id: volo | |
| run: | | |
| set +e | |
| uv run volo ci ./.volo/recordings/calc.json \ | |
| --agent examples.calc_agent:run \ | |
| --input '{"a":2,"b":3,"c":4}' \ | |
| --n 2 \ | |
| --summary-md ./volo-report.md \ | |
| --out ./.volo/reports/calc.json | |
| echo "verdict_code=$?" >> "$GITHUB_OUTPUT" | |
| - name: comment reliability report on PR | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const marker = '<!-- volo-reliability -->'; | |
| const body = marker + '\n' + fs.readFileSync('volo-report.md', 'utf8'); | |
| const { owner, repo } = context.repo; | |
| const issue_number = context.issue.number; | |
| const { data: comments } = await github.rest.issues.listComments({ owner, repo, issue_number }); | |
| const existing = comments.find(c => c.body && c.body.includes(marker)); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); | |
| } else { | |
| await github.rest.issues.createComment({ owner, repo, issue_number, body }); | |
| } | |
| - name: gate | |
| run: | | |
| test "${{ steps.volo.outputs.verdict_code }}" = "0" \ | |
| || (echo "Reliability gate failed (verdict_code=${{ steps.volo.outputs.verdict_code }})" && exit 1) |