feat(doctor): land world-class doctor surface (chokepoint, runs, undo… #565
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Browser Tests | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'tests/**' | |
| - 'src/pages/**' | |
| - 'src/pages_assets/**' | |
| - '.github/workflows/browser-tests.yml' | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - 'tests/**' | |
| - 'src/pages/**' | |
| - 'src/pages_assets/**' | |
| - '.github/workflows/browser-tests.yml' | |
| workflow_dispatch: | |
| env: | |
| NODE_VERSION: '20' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| # Install dependencies, build cass binary, and cache | |
| setup: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Clone sibling dependencies | |
| shell: bash | |
| run: | | |
| git clone --depth 1 https://github.com/Dicklesworthstone/asupersync.git ../asupersync | |
| git clone --depth 1 https://github.com/Dicklesworthstone/frankensqlite.git ../frankensqlite | |
| git clone --depth 1 https://github.com/Dicklesworthstone/franken_agent_detection.git ../franken_agent_detection | |
| git clone --depth 1 https://github.com/Dicklesworthstone/frankensearch.git ../frankensearch | |
| - name: Setup Node.js | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Cache node modules | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| id: cache-npm | |
| with: | |
| path: tests/node_modules | |
| key: ${{ runner.os }}-node-${{ hashFiles('tests/package-lock.json') }} | |
| - name: Install dependencies | |
| if: steps.cache-npm.outputs.cache-hit != 'true' | |
| working-directory: tests | |
| run: npm ci | |
| - name: Install Playwright browsers | |
| working-directory: tests | |
| run: npx playwright install --with-deps | |
| - name: Install Rust nightly | |
| uses: dtolnay/rust-toolchain@881ba7bf39a41cda34ac9e123fb41b44ed08232f # nightly | |
| - name: Cache cargo | |
| uses: Swatinem/rust-cache@ad397744b0d591a723ab90405b7247fac0e6b8db # v2 | |
| - name: Build cass binary | |
| run: cargo build --release | |
| - name: Upload cass binary | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| with: | |
| name: cass-binary | |
| path: target/release/cass | |
| retention-days: 1 | |
| # Run tests on Chromium | |
| test-chromium: | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Restore node modules | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| id: cache-npm | |
| with: | |
| path: tests/node_modules | |
| key: ${{ runner.os }}-node-${{ hashFiles('tests/package-lock.json') }} | |
| - name: Install dependencies | |
| if: steps.cache-npm.outputs.cache-hit != 'true' | |
| working-directory: tests | |
| run: npm ci | |
| - name: Install Playwright browsers | |
| working-directory: tests | |
| run: npx playwright install chromium --with-deps | |
| - name: Download cass binary | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 | |
| with: | |
| name: cass-binary | |
| path: target/release | |
| - name: Make cass binary executable | |
| run: chmod +x target/release/cass | |
| - name: Run Chromium tests | |
| working-directory: tests | |
| run: npm run test:e2e:chromium | |
| env: | |
| CI: true | |
| - name: Upload test results | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| if: always() | |
| with: | |
| name: chromium-test-results | |
| path: tests/test-results/ | |
| retention-days: 7 | |
| - name: Upload E2E report | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| if: always() | |
| with: | |
| name: chromium-e2e-report | |
| path: | | |
| tests/e2e-report/ | |
| tests/e2e-results.json | |
| retention-days: 7 | |
| # Run tests on Firefox | |
| test-firefox: | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Restore node modules | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| id: cache-npm | |
| with: | |
| path: tests/node_modules | |
| key: ${{ runner.os }}-node-${{ hashFiles('tests/package-lock.json') }} | |
| - name: Install dependencies | |
| if: steps.cache-npm.outputs.cache-hit != 'true' | |
| working-directory: tests | |
| run: npm ci | |
| - name: Install Playwright browsers | |
| working-directory: tests | |
| run: npx playwright install firefox --with-deps | |
| - name: Download cass binary | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 | |
| with: | |
| name: cass-binary | |
| path: target/release | |
| - name: Make cass binary executable | |
| run: chmod +x target/release/cass | |
| - name: Run Firefox tests | |
| working-directory: tests | |
| run: npm run test:e2e:firefox | |
| env: | |
| CI: true | |
| - name: Upload test results | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| if: always() | |
| with: | |
| name: firefox-test-results | |
| path: tests/test-results/ | |
| retention-days: 7 | |
| - name: Upload E2E report | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| if: always() | |
| with: | |
| name: firefox-e2e-report | |
| path: | | |
| tests/e2e-report/ | |
| tests/e2e-results.json | |
| retention-days: 7 | |
| # Run tests on WebKit (Safari) | |
| test-webkit: | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Restore node modules | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| id: cache-npm | |
| with: | |
| path: tests/node_modules | |
| key: ${{ runner.os }}-node-${{ hashFiles('tests/package-lock.json') }} | |
| - name: Install dependencies | |
| if: steps.cache-npm.outputs.cache-hit != 'true' | |
| working-directory: tests | |
| run: npm ci | |
| - name: Install Playwright browsers | |
| working-directory: tests | |
| run: npx playwright install webkit --with-deps | |
| - name: Download cass binary | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 | |
| with: | |
| name: cass-binary | |
| path: target/release | |
| - name: Make cass binary executable | |
| run: chmod +x target/release/cass | |
| - name: Run WebKit tests | |
| working-directory: tests | |
| run: npm run test:e2e:webkit | |
| env: | |
| CI: true | |
| - name: Upload test results | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| if: always() | |
| with: | |
| name: webkit-test-results | |
| path: tests/test-results/ | |
| retention-days: 7 | |
| - name: Upload E2E report | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| if: always() | |
| with: | |
| name: webkit-e2e-report | |
| path: | | |
| tests/e2e-report/ | |
| tests/e2e-results.json | |
| retention-days: 7 | |
| # Run mobile emulation tests | |
| test-mobile: | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Restore node modules | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 | |
| id: cache-npm | |
| with: | |
| path: tests/node_modules | |
| key: ${{ runner.os }}-node-${{ hashFiles('tests/package-lock.json') }} | |
| - name: Install dependencies | |
| if: steps.cache-npm.outputs.cache-hit != 'true' | |
| working-directory: tests | |
| run: npm ci | |
| - name: Install Playwright browsers | |
| working-directory: tests | |
| run: npx playwright install chromium webkit --with-deps | |
| - name: Download cass binary | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 | |
| with: | |
| name: cass-binary | |
| path: target/release | |
| - name: Make cass binary executable | |
| run: chmod +x target/release/cass | |
| - name: Run mobile tests | |
| working-directory: tests | |
| run: npm run test:e2e -- --project=mobile-chrome --project=mobile-safari | |
| env: | |
| CI: true | |
| - name: Upload test results | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| if: always() | |
| with: | |
| name: mobile-test-results | |
| path: tests/test-results/ | |
| retention-days: 7 | |
| - name: Upload E2E report | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| if: always() | |
| with: | |
| name: mobile-e2e-report | |
| path: | | |
| tests/e2e-report/ | |
| tests/e2e-results.json | |
| retention-days: 7 | |
| # Summary job | |
| test-summary: | |
| needs: [test-chromium, test-firefox, test-webkit, test-mobile] | |
| runs-on: ubuntu-latest | |
| if: always() | |
| timeout-minutes: 5 | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| steps: | |
| - name: Download test result artifacts | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 | |
| with: | |
| pattern: '*-test-results' | |
| path: artifacts | |
| merge-multiple: true | |
| - name: Aggregate E2E JSONL logs | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| mkdir -p test-results/e2e | |
| find artifacts -type f -path "*/test-results/e2e/*.jsonl" -print0 | sort -z | xargs -0 cat > test-results/e2e/combined.jsonl || true | |
| - name: Generate E2E summary report | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| python3 - <<'PY' | |
| import json | |
| import os | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| combined_path = Path("test-results/e2e/combined.jsonl") | |
| summary_path = Path("test-results/e2e/summary.md") | |
| summary_path.parent.mkdir(parents=True, exist_ok=True) | |
| total = passed = failed = skipped = flaky = 0 | |
| durations = {} | |
| failures = [] | |
| if combined_path.exists(): | |
| for line in combined_path.read_text(encoding="utf-8").splitlines(): | |
| line = line.strip() | |
| if not line: | |
| continue | |
| try: | |
| event = json.loads(line) | |
| except json.JSONDecodeError: | |
| continue | |
| if event.get("event") != "test_end": | |
| continue | |
| runner = event.get("runner", "unknown") | |
| result = event.get("result", {}) | |
| status = result.get("status", "unknown") | |
| duration_ms = result.get("duration_ms", 0) | |
| durations[runner] = durations.get(runner, 0) + int(duration_ms or 0) | |
| total += 1 | |
| if status == "pass": | |
| passed += 1 | |
| elif status == "skip": | |
| skipped += 1 | |
| else: | |
| failed += 1 | |
| retries = result.get("retries") | |
| if status == "pass" and retries and int(retries) > 0: | |
| flaky += 1 | |
| if status == "fail": | |
| test = event.get("test", {}) | |
| error = event.get("error", {}) | |
| failures.append({ | |
| "runner": runner, | |
| "suite": test.get("suite", "unknown"), | |
| "name": test.get("name", "unknown"), | |
| "file": test.get("file"), | |
| "line": test.get("line"), | |
| "message": error.get("message", "unknown error"), | |
| }) | |
| now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") | |
| lines = [ | |
| "# E2E Log Summary", | |
| "", | |
| f"**Generated:** {now}", | |
| f"**Combined Log:** {combined_path.as_posix()}", | |
| "", | |
| "## Totals", | |
| "", | |
| f"- **Total Tests:** {total}", | |
| f"- **Passed:** {passed}", | |
| f"- **Failed:** {failed}", | |
| f"- **Skipped:** {skipped}", | |
| f"- **Flaky (passed on retry):** {flaky}", | |
| "", | |
| "## Duration by Runner", | |
| "", | |
| "| Runner | Duration (ms) |", | |
| "|--------|---------------|", | |
| ] | |
| if durations: | |
| for runner, duration in sorted(durations.items()): | |
| lines.append(f"| {runner} | {duration} |") | |
| else: | |
| lines.append("| (none) | 0 |") | |
| lines.append("") | |
| lines.append("## Failed Tests") | |
| lines.append("") | |
| if failures: | |
| for f in failures: | |
| location = "" | |
| if f.get("file"): | |
| if f.get("line"): | |
| location = f"{f['file']}:{f['line']}" | |
| else: | |
| location = f"{f['file']}" | |
| detail = f"{f['runner']} :: {f['suite']} :: {f['name']}" | |
| if location: | |
| detail += f" ({location})" | |
| detail += f" — {f['message']}" | |
| lines.append(f"- {detail}") | |
| else: | |
| lines.append("- None") | |
| summary_path.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| print(f"Wrote {summary_path}") | |
| PY | |
| - name: Upload aggregated E2E logs | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 | |
| if: always() | |
| with: | |
| name: e2e-log-summary | |
| path: | | |
| test-results/e2e/combined.jsonl | |
| test-results/e2e/summary.md | |
| retention-days: 14 | |
| - name: Comment summary on PR | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const marker = '<!-- cass-e2e-summary -->'; | |
| const pwStart = '<!-- cass-e2e-playwright:start -->'; | |
| const pwEnd = '<!-- cass-e2e-playwright:end -->'; | |
| let summary = fs.readFileSync('test-results/e2e/summary.md', 'utf8'); | |
| if (summary.startsWith('# ')) { | |
| summary = summary.replace(/^#\s+.*$/m, '## Playwright E2E Summary'); | |
| } else if (!summary.startsWith('## ')) { | |
| summary = `## Playwright E2E Summary\n\n${summary}`; | |
| } | |
| const pwSection = `${pwStart}\n${summary.trim()}\n${pwEnd}`; | |
| const { owner, repo } = context.repo; | |
| const issue_number = context.issue.number; | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner, | |
| repo, | |
| issue_number, | |
| per_page: 100, | |
| }); | |
| const existing = comments.find(comment => comment.body.includes(marker)); | |
| const upsertSection = (body, section) => { | |
| if (body.includes(pwStart) && body.includes(pwEnd)) { | |
| const regex = new RegExp(`${pwStart}[\\s\\S]*?${pwEnd}`, 'm'); | |
| return body.replace(regex, section); | |
| } | |
| return `${body.trim()}\n\n${section}`; | |
| }; | |
| let body = existing ? existing.body : marker; | |
| if (!body.includes(marker)) { | |
| body = `${marker}\n${body}`; | |
| } | |
| body = upsertSection(body, pwSection); | |
| if (existing) { | |
| await github.rest.issues.updateComment({ | |
| owner, | |
| repo, | |
| comment_id: existing.id, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number, | |
| body, | |
| }); | |
| } | |
| - name: Check test results | |
| run: | | |
| if [[ "${{ needs.test-chromium.result }}" == "failure" ]]; then | |
| echo "Chromium tests failed" | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.test-firefox.result }}" == "failure" ]]; then | |
| echo "Firefox tests failed" | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.test-webkit.result }}" == "failure" ]]; then | |
| echo "WebKit tests failed" | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.test-mobile.result }}" == "failure" ]]; then | |
| echo "Mobile tests failed" | |
| exit 1 | |
| fi | |
| echo "All browser tests passed!" |