ci: temporary backfill workflow for /compatibility ingest #1
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Temporary workflow: re-run the `Generate report` + `Submit results to | |
| # /compatibility ingest` steps from .github/workflows/nextjs-deploy-suite.yml | |
| # against the artifacts of a previously-completed nightly run whose ingest | |
| # submission was broken (e.g. run 25925392641, which hit a DNS error before | |
| # the URL fix landed). | |
| # | |
| # The steps below are intentionally copied byte-for-byte from | |
| # nextjs-deploy-suite.yml's `report` job. The only difference is the artifact | |
| # source: instead of `actions/download-artifact` pulling from the current | |
| # run's matrix, we use the `run-id:` field to pull from the target run. | |
| # | |
| # Delete this workflow (and its branch) once the backfill is done. | |
| name: Backfill /compatibility ingest | |
| on: | |
| push: | |
| branches: | |
| - opencode/backfill-compat-ingest | |
| workflow_dispatch: | |
| inputs: | |
| run-id: | |
| description: nextjs-deploy-suite run id to reprocess | |
| required: true | |
| type: string | |
| permissions: | |
| contents: read | |
| actions: read | |
| jobs: | |
| backfill: | |
| name: Backfill ingest for ${{ inputs.run-id || '25925392641' }} | |
| if: github.repository == 'cloudflare/vinext' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Download all test results | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: test-results-* | |
| path: results | |
| merge-multiple: true | |
| # Pull from the specified historical run instead of the current one. | |
| run-id: ${{ inputs.run-id || '25925392641' }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Verbatim copy of the `Generate report` step from nextjs-deploy-suite.yml. | |
| # Producing `report/compat-ingest.json` is what we actually need; the | |
| # rest (job summary, failed-tests table) is harmless on a backfill. | |
| - name: Generate report | |
| id: report | |
| uses: actions/github-script@v7 | |
| env: | |
| # Use the source run's id as the runKey so the upsert lands on the | |
| # same row the original submission would have produced. | |
| SOURCE_RUN_ID: ${{ inputs.run-id || '25925392641' }} | |
| VINEXT_REF: main | |
| NEXT_REF: v16.2.6 | |
| SUITE_FILTER: all | |
| with: | |
| script: | | |
| const fs = require('node:fs'); | |
| const path = require('node:path'); | |
| function findResultFiles(dir) { | |
| const files = []; | |
| if (!fs.existsSync(dir)) return files; | |
| for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { | |
| const full = path.join(dir, entry.name); | |
| if (entry.isDirectory()) { | |
| files.push(...findResultFiles(full)); | |
| } else if (entry.name.endsWith('.results.json')) { | |
| files.push(full); | |
| } | |
| } | |
| return files; | |
| } | |
| const resultFiles = findResultFiles('results'); | |
| if (resultFiles.length === 0) { | |
| core.summary.addHeading('Next.js Deploy Suite', 2); | |
| core.summary.addRaw('No test result files found. All shards may have been skipped or produced no output.'); | |
| await core.summary.write(); | |
| return; | |
| } | |
| const passed = []; | |
| const failed = []; | |
| const skipped = []; | |
| // Per-test-file counts for the /compatibility page ingest. | |
| const fileCounts = new Map(); | |
| function bumpFile(suite, key) { | |
| let cur = fileCounts.get(suite); | |
| if (!cur) { | |
| cur = { suite, total: 0, passed: 0, failed: 0, skipped: 0 }; | |
| fileCounts.set(suite, cur); | |
| } | |
| cur.total++; | |
| cur[key]++; | |
| } | |
| // Derive a canonical Next.js test path from the result-file path. | |
| // | |
| // Next.js's run-tests.js does NOT populate testResults[].testFilePath | |
| // in the JSON, so we have to recover it from where the file lives in | |
| // the artifact tree. Each shard uploads files at their relative path | |
| // inside next.js/test/, e.g. `e2e/app-dir/foo/foo.test.ts.results.json`. | |
| // | |
| // The report job uses `merge-multiple: true`, which flattens the | |
| // contents of every artifact into `results/`. We also defensively | |
| // strip a leading `test-results-N/` segment in case that ever | |
| // changes and shards get nested under their artifact names. | |
| // Result: a canonical path of the form `test/e2e/app-dir/foo/foo.test.ts`. | |
| function deriveSuiteName(file) { | |
| let rel = path.relative('results', file); | |
| // Defensive: strip a leading `test-results-<n>/` shard prefix if | |
| // the download layout ever changes. | |
| rel = rel.replace(/^test-results-[^/]+\//, ''); | |
| const stripped = rel.replace(/\.results\.json$/, ''); | |
| if (stripped && stripped.includes('/')) { | |
| return `test/${stripped}`; | |
| } | |
| // Fallback: file sits at the top level of results/ (shouldn't | |
| // happen in practice). Use the basename so we don't break ingest. | |
| return path.basename(file, '.results.json'); | |
| } | |
| for (const file of resultFiles) { | |
| try { | |
| const data = JSON.parse(fs.readFileSync(file, 'utf8')); | |
| const suiteName = deriveSuiteName(file); | |
| for (const suite of data.testResults || []) { | |
| for (const tc of suite.assertionResults || []) { | |
| const testName = tc.ancestorTitles | |
| ? [...tc.ancestorTitles, tc.title].join(' > ') | |
| : tc.fullName || tc.title; | |
| if (tc.status === 'passed') { | |
| passed.push({ suite: suiteName, test: testName }); | |
| bumpFile(suiteName, 'passed'); | |
| } else if (tc.status === 'failed') { | |
| const msg = (tc.failureMessages || []).join('\n').slice(0, 500); | |
| failed.push({ suite: suiteName, test: testName, message: msg }); | |
| bumpFile(suiteName, 'failed'); | |
| } else { | |
| skipped.push({ suite: suiteName, test: testName }); | |
| bumpFile(suiteName, 'skipped'); | |
| } | |
| } | |
| } | |
| } catch (e) { | |
| core.warning(`Failed to parse ${file}: ${e.message}`); | |
| } | |
| } | |
| // Persist the per-file counts for the next step to submit to D1. | |
| // Use the SOURCE run id (not this backfill workflow's run id) as | |
| // the runKey, so the row upserts onto whatever the original | |
| // submission would have written. | |
| const sourceRunId = process.env.SOURCE_RUN_ID; | |
| fs.mkdirSync('report', { recursive: true }); | |
| fs.writeFileSync( | |
| 'report/compat-ingest.json', | |
| JSON.stringify( | |
| { | |
| kind: 'deploy', | |
| runKey: sourceRunId, | |
| vinextRef: process.env.VINEXT_REF, | |
| nextRef: process.env.NEXT_REF, | |
| commitSha: null, | |
| files: Array.from(fileCounts.values()), | |
| }, | |
| null, | |
| 2, | |
| ) + '\n', | |
| ); | |
| const total = passed.length + failed.length + skipped.length; | |
| const passRate = total > 0 ? ((passed.length / total) * 100).toFixed(1) : '0.0'; | |
| core.summary.addHeading('Backfill', 2); | |
| core.summary.addRaw( | |
| `Reprocessed run \`${sourceRunId}\`: **${passed.length}** passed, **${failed.length}** failed, **${skipped.length}** skipped (${total} total, ${passRate}% pass rate).\n` | |
| ); | |
| await core.summary.write(); | |
| # Verbatim copy of the `Submit results to /compatibility ingest` step, | |
| # minus the suite-filter/main-branch guard (we wouldn't be running this | |
| # workflow at all if the source run wasn't a main-branch full-suite run). | |
| - name: Submit results to /compatibility ingest | |
| if: hashFiles('report/compat-ingest.json') != '' | |
| env: | |
| COMPAT_INGEST_URL: ${{ vars.COMPAT_INGEST_URL || 'https://vinext-web.vinext.workers.dev/api/compatibility' }} | |
| COMPAT_INGEST_SECRET: ${{ secrets.COMPAT_INGEST_SECRET }} | |
| run: | | |
| if [ -z "${COMPAT_INGEST_SECRET:-}" ]; then | |
| echo "::error::COMPAT_INGEST_SECRET is not configured." | |
| exit 1 | |
| fi | |
| echo "Submitting compat results to ${COMPAT_INGEST_URL}" | |
| if ! curl --silent --show-error --fail-with-body \ | |
| -X POST "${COMPAT_INGEST_URL}" \ | |
| -H "Content-Type: application/json" \ | |
| -H "X-Compat-Secret: ${COMPAT_INGEST_SECRET}" \ | |
| --data-binary @report/compat-ingest.json; then | |
| echo "::error::Compatibility ingest POST failed." | |
| exit 1 | |
| fi |