diff --git a/package.json b/package.json index a8a0207acb..2cd46cc778 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "security:vite-env-secrets": "node scripts/check-vite-env-secrets.mjs", "agent:preflight": "node scripts/agent-preflight.mjs", "agent:pr-snapshot": "node scripts/agent-pr-snapshot.mjs", + "agent:attach-pr-evidence": "node scripts/publish-pr-media-evidence.mjs", "worktree:bootstrap": "node scripts/bootstrap-worktree.mjs", "worktree:bootstrap:test-only": "node scripts/bootstrap-worktree.mjs --ignore-scripts", "worktree:env": "node scripts/bootstrap-worktree.mjs --skip-install", diff --git a/scripts/publish-pr-media-evidence.mjs b/scripts/publish-pr-media-evidence.mjs new file mode 100644 index 0000000000..1cef329211 --- /dev/null +++ b/scripts/publish-pr-media-evidence.mjs @@ -0,0 +1,424 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { constants } from 'node:fs'; +import { lstat, mkdtemp, open, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { promisify } from 'node:util'; +import { pathToFileURL } from 'node:url'; +import { titleFromFileName } from './playwright-screenshot-gallery.mjs'; + +const execFileAsync = promisify(execFile); +const MIN_GH_VERSION = [2, 99, 0]; +const MAX_ATTACHMENTS = 50; +const MAX_ATTACHMENT_BYTES = 10 * 1024 * 1024; +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); +const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const FILE_NAME_PATTERN = /^[A-Za-z0-9._-]+$/; +const SHA_PATTERN = /^[0-9a-f]{40}$/i; + +/** + * Validated evidence that is safe to publish to one pull request. + * + * @typedef {object} PrMediaEvidence + * @property {string} repository + * @property {number} prNumber + * @property {string} testedHeadSha + * @property {{id: number, attempt: number, url: string}} run + * @property {Array<{path: string, filename: string, byteSize: number, alt: string}>} attachments + */ + +export function parseArgs(argv = []) { + const result = { prNumber: null, runId: null }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const readValue = () => { + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`${arg} requires a value`); + index += 1; + return value; + }; + + if (arg === '--pr') result.prNumber = parsePositiveInteger(readValue(), '--pr'); + else if (arg.startsWith('--pr=')) result.prNumber = parsePositiveInteger(arg.slice(5), '--pr'); + else if (arg === '--run-id') result.runId = parsePositiveInteger(readValue(), '--run-id'); + else if (arg.startsWith('--run-id=')) result.runId = parsePositiveInteger(arg.slice(9), '--run-id'); + else throw new Error(`Unknown argument: ${arg}`); + } + + if (result.prNumber === null) throw new Error('--pr is required'); + if (result.runId === null) throw new Error('--run-id is required'); + return result; +} + +function parsePositiveInteger(value, flag) { + if (!/^[1-9]\d*$/.test(String(value))) throw new Error(`${flag} must be a positive integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) throw new Error(`${flag} must be a safe positive integer`); + return parsed; +} + +function matchGhVersion(output) { + const match = String(output).match(/\bgh version (\d+)\.(\d+)\.(\d+)(-([^+\s]+))?(?:\+[^\s]+)?\b/i); + if (!match) throw new Error('Could not parse the GitHub CLI version'); + return match; +} + +export function assertSupportedGhVersion(output) { + const match = matchGhVersion(output); + const version = match.slice(1, 4).map(Number); + for (let index = 0; index < MIN_GH_VERSION.length; index += 1) { + if (version[index] > MIN_GH_VERSION[index]) return version; + if (version[index] < MIN_GH_VERSION[index]) { + throw new Error('GitHub CLI 2.99.0 or newer is required for media attachments'); + } + } + if (match[5]) throw new Error('GitHub CLI 2.99.0 or newer is required for media attachments'); + return version; +} + +export function validateRepository(value) { + const repository = String(value).trim(); + if (!REPOSITORY_PATTERN.test(repository)) { + throw new Error(`Invalid repository ${JSON.stringify(repository)}; expected owner/name`); + } + return repository; +} + +export function validateCurrentPr(pr, { repository, prNumber, testedHeadSha }) { + if (!pr || pr.number !== prNumber) throw new Error(`Pull request #${prNumber} was not returned`); + if (pr.state !== 'open') throw new Error(`Pull request #${prNumber} is not open`); + if (pr.base?.repo?.full_name !== repository) throw new Error('Pull request base repository does not match'); + if (pr.head?.repo?.full_name !== repository) throw new Error('Fork pull requests cannot receive trusted media evidence'); + if (!SHA_PATTERN.test(String(pr.head?.sha ?? ''))) throw new Error('Pull request head SHA is invalid'); + if (pr.head.sha !== testedHeadSha) throw new Error('Pull request head changed after the visual run'); + return pr; +} + +export function selectArtifact(artifactsResponse, { runId, runAttempt }) { + const expectedName = `playwright-gallery-${runId}-${runAttempt}`; + const artifacts = Array.isArray(artifactsResponse?.artifacts) ? artifactsResponse.artifacts : []; + const matches = artifacts.filter((artifact) => artifact?.name === expectedName && artifact.expired === false); + if (matches.length !== 1) { + throw new Error(`Expected one non-expired artifact named ${expectedName}; found ${matches.length}`); + } + return matches[0]; +} + +export function validateRunProvenance({ repository, prNumber, runId, workflow, run, jobs, pr }) { + validateRepository(repository); + if (!Number.isSafeInteger(workflow?.id) || workflow.id <= 0) throw new Error('E2E Visual workflow identity is invalid'); + if (run?.id !== runId) throw new Error('Actions run id does not match the requested run'); + if (run.workflow_id !== workflow.id) throw new Error('Actions run belongs to a different workflow'); + if (run.event !== 'pull_request') throw new Error('Actions run was not triggered by a pull request'); + if (run.conclusion === 'cancelled') throw new Error('Cancelled Actions runs cannot be published'); + if (run.head_repository?.full_name !== repository) throw new Error('Actions run came from a fork'); + if (!Array.isArray(run.pull_requests) || !run.pull_requests.some((candidate) => candidate?.number === prNumber)) { + throw new Error(`Actions run is not associated with pull request #${prNumber}`); + } + if (!SHA_PATTERN.test(String(run.head_sha ?? ''))) throw new Error('Actions run head SHA is invalid'); + if (!Number.isSafeInteger(run.run_attempt) || run.run_attempt <= 0) throw new Error('Actions run attempt is invalid'); + if (!isHttpsUrl(run.html_url)) throw new Error('Actions run URL must use HTTPS'); + + const jobList = Array.isArray(jobs?.jobs) ? jobs.jobs : []; + if (!jobList.some((job) => job?.name === 'chrome-gallery' && job.conclusion === 'success')) { + throw new Error('The chrome-gallery job did not finish successfully'); + } + + validateCurrentPr(pr, { repository, prNumber, testedHeadSha: run.head_sha }); + return { + run: { id: runId, attempt: run.run_attempt, url: run.html_url }, + testedHeadSha: run.head_sha, + }; +} + +export function validateProvenance(input) { + const provenance = validateRunProvenance(input); + return { + ...provenance, + artifact: selectArtifact(input.artifacts, { + runId: input.runId, + runAttempt: provenance.run.attempt, + }), + }; +} + +function isHttpsUrl(value) { + return typeof value === 'string' && URL.canParse(value) && new URL(value).protocol === 'https:'; +} + +export function markerForEvidence({ repository, prNumber, testedHeadSha }) { + validateRepository(repository); + if (!Number.isSafeInteger(prNumber) || prNumber <= 0) throw new Error('Invalid pull request number'); + if (!SHA_PATTERN.test(testedHeadSha)) throw new Error('Invalid tested head SHA'); + return ``; +} + +export function commentsWithMarker(comments, { login, marker }) { + return comments.filter( + (comment) => + comment?.user?.login === login && + typeof comment.body === 'string' && + comment.body.split(/\r?\n/).includes(marker), + ); +} + +export function buildCommentBody(evidence) { + const marker = markerForEvidence(evidence); + return `${marker}\nWorldMonitor E2E visual evidence for tested head \`${evidence.testedHeadSha}\`.\n\nRun: [${evidence.run.id}, attempt ${evidence.run.attempt}](${evidence.run.url})\n\nAttached ${evidence.attachments.length} validated PNG ${evidence.attachments.length === 1 ? 'capture' : 'captures'}.\n`; +} + +export function buildCommentArgs(evidence, bodyFile) { + const args = ['pr', 'comment', String(evidence.prNumber), '--repo', evidence.repository, '--body-file', bodyFile]; + for (const attachment of evidence.attachments) { + args.push('--attach', `${attachment.path}#${attachment.alt}`); + } + return args; +} + +export async function validateAttachments(tempDirectory, fs = { lstat, open, readdir }) { + const screenshotsDirectory = path.join(tempDirectory, 'screenshots'); + const imagesDirectory = path.join(screenshotsDirectory, 'images'); + for (const directory of [tempDirectory, screenshotsDirectory, imagesDirectory]) { + const stats = await fs.lstat(directory); + if (stats.isSymbolicLink() || !stats.isDirectory()) { + throw new Error(`Attachment path component is not a real directory: ${directory}`); + } + } + + const entries = await fs.readdir(imagesDirectory, { withFileTypes: true }); + if (entries.length < 1 || entries.length > MAX_ATTACHMENTS) { + throw new Error(`Expected 1..${MAX_ATTACHMENTS} screenshot entries; found ${entries.length}`); + } + + const attachments = []; + for (const entry of entries) { + if (entry.isSymbolicLink()) throw new Error(`Symlinked screenshot is not allowed: ${entry.name}`); + if (entry.isDirectory()) throw new Error(`Nested screenshot directory is not allowed: ${entry.name}`); + if (!entry.isFile()) throw new Error(`Screenshot entry is not a regular file: ${entry.name}`); + if (entry.name.includes('#')) throw new Error(`Screenshot filename cannot contain #: ${entry.name}`); + if (!FILE_NAME_PATTERN.test(entry.name)) throw new Error(`Invalid screenshot filename: ${entry.name}`); + if (!entry.name.endsWith('.png')) throw new Error(`Screenshot must use the lowercase .png extension: ${entry.name}`); + + const absolutePath = path.resolve(imagesDirectory, entry.name); + const entryStats = await fs.lstat(absolutePath); + if (entryStats.isSymbolicLink() || !entryStats.isFile()) { + throw new Error(`Screenshot is not a regular file: ${entry.name}`); + } + + let handle; + try { + handle = await fs.open(absolutePath, constants.O_RDONLY | constants.O_NOFOLLOW); + const fileStats = await handle.stat(); + if (!fileStats.isFile()) throw new Error(`Screenshot is not a regular file: ${entry.name}`); + if (fileStats.size < 1 || fileStats.size > MAX_ATTACHMENT_BYTES) { + throw new Error(`Screenshot size is outside 1 byte..10 MiB: ${entry.name}`); + } + const signature = Buffer.alloc(PNG_SIGNATURE.length); + const { bytesRead } = await handle.read(signature, 0, signature.length, 0); + if (bytesRead !== PNG_SIGNATURE.length || !signature.equals(PNG_SIGNATURE)) { + throw new Error(`Screenshot has an invalid PNG signature: ${entry.name}`); + } + attachments.push({ + alt: `WorldMonitor E2E visual evidence: ${titleFromFileName(entry.name)}`, + byteSize: fileStats.size, + filename: entry.name, + path: absolutePath, + }); + } finally { + await handle?.close(); + } + } + + return attachments.sort((left, right) => left.filename.localeCompare(right.filename)); +} + +export function flattenSlurpedPages(value) { + if (!Array.isArray(value)) throw new Error('Expected a paginated GitHub response'); + return value.flatMap((page) => { + if (!Array.isArray(page)) throw new Error('Expected each GitHub response page to be an array'); + return page; + }); +} + +function commentIdOrder(left, right) { + const leftId = BigInt(left.id); + const rightId = BigInt(right.id); + return leftId < rightId ? -1 : leftId > rightId ? 1 : 0; +} + +export async function publishPrMediaEvidence(options, dependencies = {}) { + const runGh = dependencies.runGh ?? createGhRunner(dependencies.ghBin); + const makeTempDirectory = dependencies.mkdtemp ?? mkdtemp; + const removeTempDirectory = dependencies.rm ?? rm; + const writeTextFile = dependencies.writeFile ?? writeFile; + const tempBase = dependencies.tmpdir?.() ?? tmpdir(); + + assertSupportedGhVersion(await runGh(['--version'])); + const repository = validateRepository( + dependencies.repository ?? (await runGh(['repo', 'view', '--json', 'nameWithOwner', '--jq', '.nameWithOwner'])), + ); + const loginResponse = await ghApiJson(runGh, 'user'); + const login = String(loginResponse?.login ?? ''); + if (!login) throw new Error('Could not resolve the authenticated GitHub login'); + + const workflow = await ghApiJson(runGh, `repos/${repository}/actions/workflows/e2e-visual.yml`); + const run = await ghApiJson(runGh, `repos/${repository}/actions/runs/${options.runId}`); + const jobs = await ghApiJson(runGh, `repos/${repository}/actions/runs/${options.runId}/jobs?per_page=100`); + const prEndpoint = `repos/${repository}/pulls/${options.prNumber}`; + const pr = await ghApiJson(runGh, prEndpoint); + const provenance = validateRunProvenance({ + jobs, + pr, + prNumber: options.prNumber, + repository, + run, + runId: options.runId, + workflow, + }); + + const marker = markerForEvidence({ + prNumber: options.prNumber, + repository, + testedHeadSha: provenance.testedHeadSha, + }); + const existing = commentsWithMarker(await listComments(runGh, repository, options.prNumber), { login, marker }); + if (existing.length > 0) { + const sorted = [...existing].sort(commentIdOrder); + const retained = sorted[0]; + const deletedDuplicateIds = []; + for (const duplicate of sorted.slice(1)) { + await runGh(['api', '--method', 'DELETE', `repos/${repository}/issues/comments/${duplicate.id}`]); + deletedDuplicateIds.push(duplicate.id); + } + return { + status: 'already-published', + repository, + prNumber: options.prNumber, + testedHeadSha: provenance.testedHeadSha, + runId: options.runId, + commentId: retained.id, + deletedDuplicateIds, + }; + } + + const artifacts = await ghApiJson( + runGh, + `repos/${repository}/actions/runs/${options.runId}/artifacts?per_page=100`, + ); + const artifact = selectArtifact(artifacts, { + runAttempt: provenance.run.attempt, + runId: options.runId, + }); + + const tempDirectory = await makeTempDirectory(path.join(tempBase, 'worldmonitor-pr-media-')); + try { + await runGh([ + 'run', + 'download', + String(options.runId), + '--repo', + repository, + '--name', + artifact.name, + '--dir', + tempDirectory, + ]); + let attachments = await validateAttachments(tempDirectory, dependencies.fs); + /** @type {PrMediaEvidence} */ + const evidence = { + attachments, + prNumber: options.prNumber, + repository, + run: provenance.run, + testedHeadSha: provenance.testedHeadSha, + }; + const bodyFile = path.join(tempDirectory, 'pr-comment.md'); + await writeTextFile(bodyFile, buildCommentBody(evidence), { encoding: 'utf8', flag: 'wx' }); + + attachments = await validateAttachments(tempDirectory, dependencies.fs); + evidence.attachments = attachments; + validateCurrentPr(await ghApiJson(runGh, prEndpoint), { + prNumber: options.prNumber, + repository, + testedHeadSha: evidence.testedHeadSha, + }); + await runGh(buildCommentArgs(evidence, bodyFile)); + + const matches = commentsWithMarker(await listComments(runGh, repository, options.prNumber), { login, marker }) + .sort(commentIdOrder); + if (matches.length === 0) throw new Error('Created comment could not be confirmed'); + const retained = matches[0]; + const deletedDuplicateIds = []; + for (const duplicate of matches.slice(1)) { + await runGh(['api', '--method', 'DELETE', `repos/${repository}/issues/comments/${duplicate.id}`]); + deletedDuplicateIds.push(duplicate.id); + } + + return { + status: 'published', + repository, + prNumber: options.prNumber, + testedHeadSha: evidence.testedHeadSha, + runId: options.runId, + commentId: retained.id, + attachmentCount: evidence.attachments.length, + deletedDuplicateIds, + }; + } finally { + await removeTempDirectory(tempDirectory, { recursive: true, force: true }); + } +} + +export function createGhRunner(ghBin = 'gh') { + return async (args) => { + const { stdout } = await execFileAsync(ghBin, args, { + encoding: 'utf8', + maxBuffer: 16 * 1024 * 1024, + shell: false, + }); + return stdout; + }; +} + +async function ghApiJson(runGh, endpoint) { + const raw = await runGh(['api', endpoint]); + try { + return JSON.parse(raw); + } catch { + throw new Error(`GitHub API returned invalid JSON for ${endpoint}`); + } +} + +async function listComments(runGh, repository, prNumber) { + const endpoint = `repos/${repository}/issues/${prNumber}/comments?per_page=100`; + const raw = await runGh(['api', '--paginate', '--slurp', endpoint]); + try { + return flattenSlurpedPages(JSON.parse(raw)); + } catch (error) { + if (error instanceof SyntaxError) throw new Error(`GitHub API returned invalid JSON for ${endpoint}`); + throw error; + } +} + +async function main() { + const result = await publishPrMediaEvidence(parseArgs(process.argv.slice(2))); + process.stdout.write(`${JSON.stringify(result)}\n`); +} + +const invokedDirectly = Boolean(process.argv[1]) && import.meta.url === pathToFileURL(process.argv[1]).href; +if (invokedDirectly) { + // Terminal success marker. Emitted from .then() so it can ONLY print after main() has fully + // resolved — a throw anywhere inside, including a late publish step, skips it. Any marker + // written INSIDE main() would print before later work and could vouch for a run that then + // died (exactly how #6092 stayed invisible). Format mirrors runSeed() so the crash + // diagnostic recognises it; without it a clean run is indistinguishable from a silent death. + const __runStartedAt = Date.now(); + main() + .then(() => console.log(`\n=== Done (${Date.now() - __runStartedAt}ms) ===`)) + .catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/tests/e2e-visual-workflow.test.mjs b/tests/e2e-visual-workflow.test.mjs index b3b4c341c3..8e712ecb3e 100644 --- a/tests/e2e-visual-workflow.test.mjs +++ b/tests/e2e-visual-workflow.test.mjs @@ -78,18 +78,34 @@ describe('E2E visual workflow contract', () => { }); it('uploads each Playwright run immediately with the #6496 artifact contract', () => { + const allRawArtifactNames = []; for (const [jobId, job] of Object.entries(visual.jobs)) { - const playwrightRuns = jobSteps(job).filter((step) => - /npm run test:e2e:/.test(String(step.run ?? '')), - ); + const steps = jobSteps(job); + const playwrightRuns = steps + .map((step, index) => ({ index, step })) + .filter(({ step }) => /npm run test:e2e:/.test(String(step.run ?? ''))); assert.ok(playwrightRuns.length > 0, `${jobId} must invoke playwright`); - const uploads = jobSteps(job).filter((step) => stepUses(step).startsWith('actions/upload-artifact@')); + const uploads = steps.filter((step) => stepUses(step).startsWith('actions/upload-artifact@')); assert.ok(uploads.length >= playwrightRuns.length, `${jobId} must upload after each playwright run`); - for (const upload of uploads) { + const rawArtifactNames = []; + for (const [runOffset, playwrightRun] of playwrightRuns.entries()) { + const nextRunIndex = playwrightRuns[runOffset + 1]?.index ?? steps.length; + const upload = steps.slice(playwrightRun.index + 1, nextRunIndex).find((step) => { + if (!stepUses(step).startsWith('actions/upload-artifact@')) return false; + const uploadPath = Array.isArray(step.with?.path) + ? step.with.path + : String(step.with?.path ?? '').split('\n'); + return uploadPath.some((entry) => /^test-results\/?$/.test(String(entry).trim())); + }); + assert.ok(upload, `${jobId} must upload raw test-results after Playwright run ${runOffset + 1}`); assert.match(String(upload.if), /!cancelled\(\)|always\(\)/); assert.match(String(upload.with.name), /github\.run_attempt/); + assert.doesNotMatch(String(upload.with.name), /gallery/i, 'gallery upload is not raw test-results evidence'); + rawArtifactNames.push(String(upload.with.name)); + allRawArtifactNames.push(String(upload.with.name)); + const path = upload.with.path; const flattened = Array.isArray(path) ? path @@ -98,12 +114,22 @@ describe('E2E visual workflow contract', () => { .map((line) => line.trim()) .filter(Boolean); assert.ok( - flattened.some((entry) => entry === 'test-results' || entry === 'test-results/' || entry === 'gallery/'), - `${jobId} upload must include test-results or gallery, got ${JSON.stringify(path)}`, + flattened.some((entry) => entry === 'test-results' || entry === 'test-results/'), + `${jobId} raw upload must include test-results, got ${JSON.stringify(path)}`, ); assert.match(stepUses(upload), /@[0-9a-f]{40}$/i); } + assert.equal( + new Set(rawArtifactNames).size, + rawArtifactNames.length, + `${jobId} raw Playwright uploads must have distinct names`, + ); } + assert.equal( + new Set(allRawArtifactNames).size, + allRawArtifactNames.length, + 'raw Playwright uploads must have distinct names across the workflow', + ); }); it('publishes to object storage only when the screenshot bucket is configured', () => { diff --git a/tests/publish-pr-media-evidence.test.mjs b/tests/publish-pr-media-evidence.test.mjs new file mode 100644 index 0000000000..5ea51ff468 --- /dev/null +++ b/tests/publish-pr-media-evidence.test.mjs @@ -0,0 +1,372 @@ +import assert from 'node:assert/strict'; +import { mkdir, mkdtemp, open, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { describe, it } from 'node:test'; +import { + assertSupportedGhVersion, + buildCommentArgs, + commentsWithMarker, + markerForEvidence, + parseArgs, + publishPrMediaEvidence, + validateAttachments, + validateProvenance, +} from '../scripts/publish-pr-media-evidence.mjs'; + +const SHA = 'a'.repeat(40); +const REPOSITORY = 'koala73/worldmonitor'; +const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +function provenanceFixture(overrides = {}) { + const fixture = { + artifacts: { + artifacts: [{ id: 501, name: 'playwright-gallery-1234-2', expired: false }], + }, + jobs: { jobs: [{ id: 91, name: 'chrome-gallery', conclusion: 'success' }] }, + pr: { + number: 42, + state: 'open', + base: { repo: { full_name: REPOSITORY } }, + head: { repo: { full_name: REPOSITORY }, sha: SHA }, + }, + prNumber: 42, + repository: REPOSITORY, + run: { + id: 1234, + workflow_id: 77, + event: 'pull_request', + conclusion: 'success', + head_repository: { full_name: REPOSITORY }, + head_sha: SHA, + html_url: 'https://github.com/koala73/worldmonitor/actions/runs/1234', + pull_requests: [{ number: 42 }], + run_attempt: 2, + }, + runId: 1234, + workflow: { id: 77, path: '.github/workflows/e2e-visual.yml' }, + }; + return { ...fixture, ...overrides }; +} + +async function makeGallery(parentPrefix = 'worldmonitor-pr-media-test-') { + const root = await mkdtemp(path.join(tmpdir(), parentPrefix)); + await mkdir(path.join(root, 'screenshots', 'images'), { recursive: true }); + return root; +} + +async function writePng(root, filename, suffix = Buffer.alloc(0)) { + await writeFile(path.join(root, 'screenshots', 'images', filename), Buffer.concat([PNG_SIGNATURE, suffix])); +} + +describe('PR media command arguments and GitHub CLI gate', () => { + it('parses only positive integer PR and run ids', () => { + assert.deepEqual(parseArgs(['--pr', '42', '--run-id=1234']), { prNumber: 42, runId: 1234 }); + assert.throws(() => parseArgs(['--pr', '0', '--run-id', '1']), /positive integer/); + assert.throws(() => parseArgs(['--pr', '1']), /--run-id is required/); + assert.throws(() => parseArgs(['--pr', '1', '--run-id', '2', '--repo', 'other/repo']), /Unknown argument/); + }); + + it('requires GitHub CLI 2.99.0 or newer', () => { + assert.deepEqual(assertSupportedGhVersion('gh version 2.99.0 (2026-01-01)'), [2, 99, 0]); + assert.deepEqual(assertSupportedGhVersion('gh version 3.0.0'), [3, 0, 0]); + assert.throws(() => assertSupportedGhVersion('gh version 2.98.9'), /2\.99\.0 or newer/); + assert.throws(() => assertSupportedGhVersion('gh version 2.99.0-rc.1'), /2\.99\.0 or newer/); + assert.throws(() => assertSupportedGhVersion('not gh'), /Could not parse/); + }); +}); + +describe('trusted Actions provenance', () => { + it('accepts the exact workflow, same-repository PR head, successful gallery job, and one artifact', () => { + const result = validateProvenance(provenanceFixture()); + assert.equal(result.testedHeadSha, SHA); + assert.deepEqual(result.run, { + id: 1234, + attempt: 2, + url: 'https://github.com/koala73/worldmonitor/actions/runs/1234', + }); + assert.equal(result.artifact.id, 501); + }); + + it('rejects the wrong workflow, fork, closed PR, stale head, and failed chrome job', () => { + const wrongWorkflow = provenanceFixture(); + wrongWorkflow.run.workflow_id = 78; + assert.throws(() => validateProvenance(wrongWorkflow), /different workflow/); + + const fork = provenanceFixture(); + fork.run.head_repository.full_name = 'contributor/worldmonitor'; + assert.throws(() => validateProvenance(fork), /fork/); + + const forkPr = provenanceFixture(); + forkPr.pr.head.repo.full_name = 'contributor/worldmonitor'; + assert.throws(() => validateProvenance(forkPr), /Fork pull requests/); + + const closed = provenanceFixture(); + closed.pr.state = 'closed'; + assert.throws(() => validateProvenance(closed), /not open/); + + const stale = provenanceFixture(); + stale.pr.head.sha = 'b'.repeat(40); + assert.throws(() => validateProvenance(stale), /head changed/); + + const failedJob = provenanceFixture(); + failedJob.jobs.jobs[0].conclusion = 'failure'; + assert.throws(() => validateProvenance(failedJob), /did not finish successfully/); + }); + + it('rejects expired and ambiguous gallery artifacts', () => { + const expired = provenanceFixture(); + expired.artifacts.artifacts[0].expired = true; + assert.throws(() => validateProvenance(expired), /found 0/); + + const ambiguous = provenanceFixture(); + ambiguous.artifacts.artifacts.push({ + id: 502, + name: 'playwright-gallery-1234-2', + expired: false, + }); + assert.throws(() => validateProvenance(ambiguous), /found 2/); + }); +}); + +describe('PNG attachment boundary', () => { + it('sorts valid PNGs and derives stable explicit alt text', async () => { + const root = await makeGallery('worldmonitor media with spaces-'); + try { + await writePng(root, '010-second_view.png'); + await writePng(root, '002-first-view.png'); + const attachments = await validateAttachments(root); + assert.deepEqual( + attachments.map(({ filename, alt, byteSize }) => ({ filename, alt, byteSize })), + [ + { + filename: '002-first-view.png', + alt: 'WorldMonitor E2E visual evidence: first view', + byteSize: 8, + }, + { + filename: '010-second_view.png', + alt: 'WorldMonitor E2E visual evidence: second view', + byteSize: 8, + }, + ], + ); + assert.ok(attachments.every((attachment) => path.isAbsolute(attachment.path))); + + const evidence = { + repository: REPOSITORY, + prNumber: 42, + testedHeadSha: SHA, + run: { id: 1234, attempt: 2, url: 'https://example.test/run' }, + attachments, + }; + const bodyFile = path.join(root, 'comment body.md'); + const args = buildCommentArgs(evidence, bodyFile); + assert.deepEqual(args.slice(0, 8), [ + 'pr', + 'comment', + '42', + '--repo', + REPOSITORY, + '--body-file', + bodyFile, + '--attach', + ]); + assert.equal(args[8], `${attachments[0].path}#${attachments[0].alt}`); + assert.equal(args[9], '--attach'); + assert.equal(args[10], `${attachments[1].path}#${attachments[1].alt}`); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it('rejects bad PNG magic and zero-byte files', async () => { + const root = await makeGallery(); + try { + await writeFile(path.join(root, 'screenshots', 'images', 'bad.png'), Buffer.from('not-png!')); + await assert.rejects(validateAttachments(root), /invalid PNG signature/); + await writeFile(path.join(root, 'screenshots', 'images', 'bad.png'), Buffer.alloc(0)); + await assert.rejects(validateAttachments(root), /size is outside/); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it('rejects 51 entries, symlinks, oversized files, and bad filenames', async () => { + const tooMany = await makeGallery(); + try { + await Promise.all(Array.from({ length: 51 }, (_, index) => writePng(tooMany, `${index}.png`))); + await assert.rejects(validateAttachments(tooMany), /found 51/); + } finally { + await rm(tooMany, { recursive: true, force: true }); + } + + const linked = await makeGallery(); + try { + await writePng(linked, 'target.png'); + await symlink('target.png', path.join(linked, 'screenshots', 'images', 'linked.png')); + await assert.rejects(validateAttachments(linked), /Symlinked screenshot/); + } finally { + await rm(linked, { recursive: true, force: true }); + } + + const oversized = await makeGallery(); + try { + const handle = await open(path.join(oversized, 'screenshots', 'images', 'large.png'), 'w'); + await handle.write(PNG_SIGNATURE, 0, PNG_SIGNATURE.length, 0); + await handle.truncate(10 * 1024 * 1024 + 1); + await handle.close(); + await assert.rejects(validateAttachments(oversized), /size is outside/); + } finally { + await rm(oversized, { recursive: true, force: true }); + } + + const badName = await makeGallery(); + try { + await writePng(badName, 'bad name.png'); + await assert.rejects(validateAttachments(badName), /Invalid screenshot filename/); + } finally { + await rm(badName, { recursive: true, force: true }); + } + }); + + it('rejects symlinked directory components', async () => { + const root = await mkdtemp(path.join(tmpdir(), 'worldmonitor-pr-media-link-')); + const target = await mkdtemp(path.join(tmpdir(), 'worldmonitor-pr-media-target-')); + try { + await mkdir(path.join(target, 'images')); + await writeFile(path.join(target, 'images', 'shot.png'), PNG_SIGNATURE); + await symlink(target, path.join(root, 'screenshots')); + await assert.rejects(validateAttachments(root), /path component is not a real directory/); + } finally { + await rm(root, { recursive: true, force: true }); + await rm(target, { recursive: true, force: true }); + } + }); +}); + +describe('author-scoped idempotence and write race handling', () => { + const marker = markerForEvidence({ repository: REPOSITORY, prNumber: 42, testedHeadSha: SHA }); + + it('matches only exact LF or CRLF marker lines from the authenticated author', () => { + const comments = [ + { id: 1, user: { login: 'publisher' }, body: `text\n${marker}` }, + { id: 2, user: { login: 'publisher' }, body: `text\r\n${marker}\r\nmore` }, + { id: 3, user: { login: 'other' }, body: marker }, + { id: 4, user: { login: 'publisher' }, body: marker.replace(SHA, 'b'.repeat(40)) }, + { id: 5, user: { login: 'publisher' }, body: `embedded ${marker} in prose` }, + ]; + assert.deepEqual(commentsWithMarker(comments, { login: 'publisher', marker }).map(({ id }) => id), [1, 2]); + }); + + it('returns a retry no-op before download or comment creation', async () => { + const calls = []; + const runGh = createFakeGh({ + calls, + initialComments: [{ id: 44, user: { login: 'publisher' }, body: marker }], + }); + const result = await publishPrMediaEvidence({ prNumber: 42, runId: 1234 }, { repository: REPOSITORY, runGh }); + assert.equal(result.status, 'already-published'); + assert.equal(result.commentId, 44); + assert.ok(!calls.some((args) => args[0] === 'run')); + assert.ok(!calls.some((args) => args[0] === 'pr')); + assert.ok(!calls.some((args) => String(args.at(-1)).endsWith('/artifacts?per_page=100'))); + }); + + it('reconciles existing own duplicates without downloading or creating a comment', async () => { + const calls = []; + const runGh = createFakeGh({ + calls, + initialComments: [ + { id: 40, user: { login: 'publisher' }, body: marker }, + { id: 10, user: { login: 'publisher' }, body: `context\r\n${marker}\r\n` }, + { id: 1, user: { login: 'other' }, body: marker }, + ], + }); + const result = await publishPrMediaEvidence({ prNumber: 42, runId: 1234 }, { repository: REPOSITORY, runGh }); + assert.equal(result.status, 'already-published'); + assert.equal(result.commentId, 10); + assert.deepEqual(result.deletedDuplicateIds, [40]); + assert.deepEqual( + calls.filter((args) => args[0] === 'api' && args[1] === '--method'), + [['api', '--method', 'DELETE', `repos/${REPOSITORY}/issues/comments/40`]], + ); + assert.ok(!calls.some((args) => args[0] === 'run')); + assert.ok(!calls.some((args) => args[0] === 'pr')); + assert.ok(!calls.some((args) => String(args.at(-1)).endsWith('/artifacts?per_page=100'))); + }); + + it('rechecks the live head immediately before the write', async () => { + const calls = []; + const runGh = createFakeGh({ calls, staleOnSecondPrRead: true }); + await assert.rejects( + publishPrMediaEvidence({ prNumber: 42, runId: 1234 }, { repository: REPOSITORY, runGh }), + /head changed/, + ); + assert.equal(calls.filter((args) => args[0] === 'api' && args.at(-1).endsWith('/pulls/42')).length, 2); + assert.ok(!calls.some((args) => args[0] === 'pr')); + }); + + it('keeps the lowest own marker comment and deletes only higher own duplicates', async () => { + const calls = []; + const postComments = [ + { id: 20, user: { login: 'publisher' }, body: marker }, + { id: 1, user: { login: 'other' }, body: marker }, + { id: 10, user: { login: 'publisher' }, body: `created\n${marker}` }, + { id: 30, user: { login: 'publisher' }, body: 'unrelated comment' }, + ]; + const runGh = createFakeGh({ calls, postComments }); + const result = await publishPrMediaEvidence({ prNumber: 42, runId: 1234 }, { repository: REPOSITORY, runGh }); + assert.equal(result.status, 'published'); + assert.equal(result.commentId, 10); + assert.deepEqual(result.deletedDuplicateIds, [20]); + assert.deepEqual( + calls.filter((args) => args[0] === 'api' && args[1] === '--method'), + [['api', '--method', 'DELETE', `repos/${REPOSITORY}/issues/comments/20`]], + ); + + const commentIndex = calls.findIndex((args) => args[0] === 'pr'); + const precedingRemoteCall = calls.slice(0, commentIndex).findLast((args) => args[0] === 'api'); + assert.equal(precedingRemoteCall.at(-1), `repos/${REPOSITORY}/pulls/42`); + const commentArgs = calls[commentIndex]; + assert.equal(commentArgs[0], 'pr'); + assert.ok(commentArgs.includes('--body-file')); + assert.ok(commentArgs.includes('--attach')); + }); +}); + +function createFakeGh({ calls, initialComments = [], postComments = [], staleOnSecondPrRead = false }) { + let commentsReads = 0; + let prReads = 0; + return async (args) => { + calls.push(args); + if (args[0] === '--version') return 'gh version 2.99.0 (test)'; + if (args[0] === 'run') { + const outputDirectory = args[args.indexOf('--dir') + 1]; + await mkdir(path.join(outputDirectory, 'screenshots', 'images'), { recursive: true }); + await writeFile(path.join(outputDirectory, 'screenshots', 'images', '001-dashboard.png'), PNG_SIGNATURE); + return ''; + } + if (args[0] === 'pr') return 'https://github.com/koala73/worldmonitor/pull/42#issuecomment-10'; + if (args[0] !== 'api') throw new Error(`Unexpected command: ${args.join(' ')}`); + if (args[1] === '--method') return ''; + + const endpoint = args.at(-1); + if (endpoint === 'user') return JSON.stringify({ login: 'publisher' }); + if (endpoint.endsWith('/actions/workflows/e2e-visual.yml')) return JSON.stringify({ id: 77 }); + if (endpoint.endsWith('/actions/runs/1234')) return JSON.stringify(provenanceFixture().run); + if (endpoint.endsWith('/jobs?per_page=100')) return JSON.stringify(provenanceFixture().jobs); + if (endpoint.endsWith('/pulls/42')) { + prReads += 1; + const pr = provenanceFixture().pr; + if (staleOnSecondPrRead && prReads === 2) pr.head.sha = 'b'.repeat(40); + return JSON.stringify(pr); + } + if (endpoint.endsWith('/artifacts?per_page=100')) return JSON.stringify(provenanceFixture().artifacts); + if (endpoint.endsWith('/comments?per_page=100')) { + commentsReads += 1; + return JSON.stringify([commentsReads === 1 ? initialComments : postComments]); + } + throw new Error(`Unexpected API endpoint: ${endpoint}`); + }; +}