diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8cca703 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,50 @@ +--- +# CI for the toolkit's own scripts. +# +# Override is the only triage script left here — adjudication moved to +# mattermost-test-system-io's test-system-io-ai-triage action. +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + name: Script tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 + with: + node-version: '22' + + - name: ci/test + run: | + node --test \ + scripts/triage-override.test.js + + actionlint: + name: Workflow lint + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: ci/actionlint + run: | + set -euo pipefail + INSTALLER_REF=3795ba2f6cb243eeca54c9d22e5c531cb9dcfb4a + ACTIONLINT_VERSION=1.7.12 + curl -fsSL -o download-actionlint.bash \ + "https://raw.githubusercontent.com/rhysd/actionlint/${INSTALLER_REF}/scripts/download-actionlint.bash" + bash download-actionlint.bash "${ACTIONLINT_VERSION}" + ./actionlint -shellcheck= .github/workflows/*.yml diff --git a/.github/workflows/e2e-ai-triage-override.yml b/.github/workflows/e2e-ai-triage-override.yml new file mode 100644 index 0000000..9e258bc --- /dev/null +++ b/.github/workflows/e2e-ai-triage-override.yml @@ -0,0 +1,208 @@ +--- +# Human override of an automated triage verdict. +# +# Corrections are the only ground truth this system gets — everything else is the +# triage grading its own homework. A maintainer saying "that was actually a real +# bug" is the one input that can prove a waiver wrong, and it is exactly what the +# false-green metric counts. +# +# The correction is recorded in the TSIO ledger *and* the checks are brought into +# line with what the human said. If the ledger write fails the checks are still +# updated — the maintainer's intent must be honoured — but the reply says so, +# because an unrecorded correction is a data point permanently lost. +name: E2E AI Triage Override (Reusable) + +on: + workflow_call: + inputs: + target_repo: + description: "Full repo name the PR and statuses belong to" + required: true + type: string + pr_number: + required: true + type: string + comment_body: + description: "Raw comment body, parsed for /e2e-triage-override " + required: true + type: string + status_context: + description: "Commit-status context to write, matching the triage workflow" + required: false + type: string + default: "e2e-test/ai-triage" + comment_id: + description: "Comment to react to, so the author sees it was picked up" + required: false + type: string + default: "" + sender: + description: "Who issued the override — recorded as corrected_by" + required: true + type: string + tsio_url: + required: false + type: string + default: "https://test-io.test.mattermost.com" + toolkit_ref: + description: >- + Ref of THIS repository to check out for the scripts. Defaults to main. + It cannot be derived: inside a called reusable workflow the `github` + context — including workflow_ref and workflow_sha — describes the + *caller*, so resolving from it would try to check this repo out at the + caller's branch. Callers testing an unmerged toolkit change must pass + the same ref they pinned `uses:` to. + required: false + type: string + default: "main" + secrets: + GH_TOKEN: + description: "Token for statuses, labels, and comments on target_repo" + required: true + TSIO_API_KEY: + description: "Optional. Without it the ledger write uses a minted OIDC token." + required: false + +permissions: + contents: read + # The ledger write authenticates with a minted OIDC token by default. + id-token: write + +jobs: + override: + runs-on: ubuntu-24.04 + steps: + # Check out THIS repository, not the caller's. + # + # In a reusable workflow `github.repository` is the *caller*, so a bare + # checkout cloned mattermost-mobile and the next step ran + # `node scripts/...` against a tree with no such file. MODULE_NOT_FOUND + # kills the job before the red fallback can post, so the run ends with no + # status at all — and absent is not fail-closed, it is silence. + # + # The ref comes from an input rather than from github.workflow_ref: that + # context also describes the caller, so deriving from it would ask for this + # repository at the caller's branch, which does not exist here. + - name: ci/checkout-toolkit + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: mattermost/mattermost-test-automation-toolkit + ref: ${{ inputs.toolkit_ref }} + persist-credentials: false + + - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 + with: + node-version: '22' + + # Verify the sender here rather than trusting the caller to have done it. + # + # `sender` is just a string input: this workflow is reusable, so whether it + # was gated upstream is a property of each caller, not of this file. A + # caller that forgets — or one that passes github.actor from a trigger that + # any user can fire — would let an outside contributor overturn a red E2E + # verdict and apply E2E/AI-Waived to their own PR. The permission is + # therefore checked against the repository the override targets, before + # anything is written. + - name: ci/verify-sender-can-write + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + TARGET_REPO: ${{ inputs.target_repo }} + SENDER: ${{ inputs.sender }} + COMMENT_ID: ${{ inputs.comment_id }} + PR_NUMBER: ${{ inputs.pr_number }} + run: | + set -euo pipefail + if [ -z "${SENDER}" ]; then + echo "::error::sender is empty — refusing to apply an unattributed override" + exit 1 + fi + PERM=$(gh api "repos/${TARGET_REPO}/collaborators/${SENDER}/permission" \ + --jq '.permission' 2>/dev/null || echo "none") + echo "sender ${SENDER} has '${PERM}' on ${TARGET_REPO}" + case "${PERM}" in + admin|maintain|write) ;; + *) + echo "::error::${SENDER} needs write access to ${TARGET_REPO} to override a triage verdict (has '${PERM}')" + exit 1 + ;; + esac + + # Confirm the sender against the comment itself, rather than believing + # the caller. The permission check above already keeps outsiders out, + # but sender is still a caller-supplied string, so one collaborator + # could be recorded as having overturned a verdict another collaborator + # actually overturned. Attribution that can be chosen by the caller is + # not attribution — the same reason corrected_by is taken from the + # authenticated principal rather than the request body. + if [ -z "${COMMENT_ID}" ]; then + echo "::error::comment_id is required — the override cannot be traced to a comment without it" + exit 1 + fi + COMMENT=$(gh api "repos/${TARGET_REPO}/issues/comments/${COMMENT_ID}" 2>/dev/null || echo '') + if [ -z "${COMMENT}" ]; then + echo "::error::comment ${COMMENT_ID} not found on ${TARGET_REPO}" + exit 1 + fi + COMMENT_AUTHOR=$(printf '%s' "${COMMENT}" | jq -r '.user.login // ""') + if [ "${COMMENT_AUTHOR}" != "${SENDER}" ]; then + echo "::error::comment ${COMMENT_ID} was written by '${COMMENT_AUTHOR}', not '${SENDER}'" + exit 1 + fi + # And that the comment belongs to the PR being overridden, so a comment + # from an unrelated issue cannot authorise a change here. + COMMENT_ISSUE=$(printf '%s' "${COMMENT}" | jq -r '.issue_url // ""') + if [ "${COMMENT_ISSUE##*/}" != "${PR_NUMBER}" ]; then + echo "::error::comment ${COMMENT_ID} belongs to ${COMMENT_ISSUE##*/}, not PR ${PR_NUMBER}" + exit 1 + fi + echo "verified: ${SENDER} wrote comment ${COMMENT_ID} on PR ${PR_NUMBER}" + + # Validate the TSIO origin before any credential can be sent to it. + # + # tsio_url is a caller-supplied string, and the next step sends TSIO_API_KEY + # — or a minted OIDC token — to whatever host it names. Unvalidated, that is + # a credential-exfiltration primitive rather than a wrong endpoint: a caller + # passing https://attacker.example gets the ledger token posted to it. Exact + # origins only, no suffix matching, because "endswith mattermost.com" is + # satisfied by evil-mattermost.com. + - name: ci/validate-tsio-origin + env: + TSIO_URL: ${{ inputs.tsio_url }} + run: | + set -euo pipefail + case "${TSIO_URL}" in + https://test-io.test.mattermost.com|https://staging-test-io.test.mattermost.com) ;; + *) + echo "::error::tsio_url is not an approved TSIO origin: ${TSIO_URL}" + exit 1 + ;; + esac + echo "TSIO origin approved: ${TSIO_URL}" + + - name: ci/apply-override + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + TSIO_API_KEY: ${{ secrets.TSIO_API_KEY }} + # Passed through the environment, never interpolated into the script + # arguments: a comment body is attacker-controlled text and must not be + # able to reach a shell as anything but data. + COMMENT_BODY: ${{ inputs.comment_body }} + # Same reasoning for the rest: sender is a GitHub login and the others + # are caller-supplied, so none of them are pasted into the script text. + TARGET_REPO: ${{ inputs.target_repo }} + PR_NUMBER: ${{ inputs.pr_number }} + SENDER: ${{ inputs.sender }} + COMMENT_ID: ${{ inputs.comment_id }} + TSIO_URL: ${{ inputs.tsio_url }} + STATUS_CONTEXT: ${{ inputs.status_context }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + node scripts/triage-override.js \ + --repo="${TARGET_REPO}" \ + --pr="${PR_NUMBER}" \ + --actor="${SENDER}" \ + --comment-id="${COMMENT_ID}" \ + --tsio-url="${TSIO_URL}" \ + --status-context="${STATUS_CONTEXT}" \ + --run-url="${RUN_URL}" diff --git a/.github/workflows/e2e-ai-triage.md b/.github/workflows/e2e-ai-triage.md new file mode 100644 index 0000000..83a7acf --- /dev/null +++ b/.github/workflows/e2e-ai-triage.md @@ -0,0 +1,20 @@ +# E2E AI triage — human override + +Adjudication (clustering, flake vs bug, check greening) lives in +[mattermost-test-system-io](https://github.com/mattermost/mattermost-test-system-io) +(`.github/actions/test-system-io-ai-triage`). This repository only hosts the +maintainer override path. + +## Override + +Workflow: [`e2e-ai-triage-override.yml`](./e2e-ai-triage-override.yml) + +Comment `/e2e-triage-override` on a PR (OWNER / MEMBER / COLLABORATOR) to +correct a triage verdict via the TSIO corrections API and adjust labels / +commit status. + +## Testing + +```bash +node --test scripts/triage-override.test.js +``` diff --git a/scripts/triage-override.js b/scripts/triage-override.js new file mode 100644 index 0000000..a391e14 --- /dev/null +++ b/scripts/triage-override.js @@ -0,0 +1,361 @@ +#!/usr/bin/env node +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +/* eslint-disable no-console */ + +/** + * Human override of an automated triage verdict. + * + * Corrections are the only ground truth this system ever gets. Everything else — + * confidence scores, signature weights, model verdicts — is the system grading + * its own homework. A maintainer saying "that was actually a real bug" is the + * single input that can tell us the triage is wrong, and it is what the + * false-green metric counts. + * + * So this does two things, and the order matters: + * + * 1. Record the correction in the TSIO ledger. This is the durable part; it + * survives the PR being merged and feeds the accuracy metrics that decide + * whether triage is ever allowed to gate anything. + * 2. Bring the checks into line with what the human said. + * + * If (1) fails we still do (2) — the maintainer's immediate intent must be + * honoured — but we say so loudly, because a correction that was not recorded is + * a data point permanently lost. + */ + +// AI waivers and human overrides must stay distinguishable: the false-green +// metric counts AI waivers that a human later reclassifies, so a human +// correction can never wear the AI label. The status reporter honours both. +const AI_WAIVED_LABEL = 'E2E/AI-Waived'; +const HUMAN_OVERRIDE_LABEL = 'E2E/Override'; +const DEFAULT_STATUS_CONTEXT = 'e2e-test/ai-triage'; + +const VERDICTS = new Set([ + 'PR_REGRESSION', + 'MAIN_REGRESSION', + 'FLAKY_TEST', + 'FLAKY_INFRA', + 'FLAKY_SERVER', + 'BUILD_OR_ENV_ERROR', + 'TEST_DEBT', + 'INCONCLUSIVE', +]); + +// Verdicts whose meaning is "not attributable to this change", i.e. the ones that +// justify a green. Mirrors WAIVABLE in triage-policy.js. +const WAIVABLE = new Set(['FLAKY_TEST', 'FLAKY_INFRA', 'FLAKY_SERVER', 'MAIN_REGRESSION']); + +/** + * Parse `/e2e-triage-override `. + * + * A reason is mandatory. The correction's whole value is as a labelled example + * for whoever later asks "why was the model wrong here" — a bare verdict with no + * explanation records that it was wrong while discarding the only part that says + * how. + * + * The verdict is matched case-insensitively and with dashes normalised, because + * people will type `flaky-infra` at least as often as `FLAKY_INFRA`. + */ +function parseCommand(body) { + const line = String(body || '') + .split('\n') + .map((l) => l.trim()) + .find((l) => l.startsWith('/e2e-triage-override')); + + if (!line) { + return {ok: false, error: 'no /e2e-triage-override command found'}; + } + + const rest = line.slice('/e2e-triage-override'.length).trim(); + if (!rest) { + return { + ok: false, + error: 'usage: `/e2e-triage-override ` — for example ' + + '`/e2e-triage-override PR_REGRESSION this really was broken by the change`', + }; + } + + const [rawVerdict, ...reasonParts] = rest.split(/\s+/); + const verdict = rawVerdict.toUpperCase().replace(/-/g, '_'); + + if (!VERDICTS.has(verdict)) { + return { + ok: false, + error: `\`${rawVerdict}\` is not a known verdict. One of: ${[...VERDICTS].join(', ')}`, + }; + } + + const reason = reasonParts.join(' ').trim(); + if (!reason) { + return { + ok: false, + error: 'a reason is required — the correction is only useful as a labelled ' + + 'example if it says *why* the verdict was wrong', + }; + } + + return {ok: true, verdict, reason, waivable: WAIVABLE.has(verdict)}; +} + +/** + * What the checks should look like after a correction. + * + * A human correcting to a waivable verdict is saying "this failure was not + * caused by the change", so the checks go green. Correcting to anything else is + * saying the opposite, and the waiver must be withdrawn — including the label, + * which is sticky and would otherwise keep greening later commits. + */ +// GitHub silently truncates a commit-status description at 140 characters. +// decideAfterOverride deliberately returns the maintainer's reason in full — the +// PR comment prints all of it — so the clamp belongs at the status call, and is +// named here so it is one tested rule rather than a bare slice at the call site. +const STATUS_DESCRIPTION_LIMIT = 140; + +function clampDescription(text) { + return String(text ?? '').slice(0, STATUS_DESCRIPTION_LIMIT); +} + +function decideAfterOverride(parsed) { + if (parsed.waivable) { + return { + state: 'success', + applyLabel: true, + description: `human override: ${parsed.verdict.toLowerCase().replace(/_/g, '-')} — ${parsed.reason}`, + }; + } + return { + state: 'failure', + applyLabel: false, + description: `human override: ${parsed.verdict.toLowerCase().replace(/_/g, '-')} — ${parsed.reason}`, + }; +} + +async function gh(token, method, path, body) { + const res = await fetch(`https://api.github.com${path}`, { + method, + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + 'Content-Type': 'application/json', + }, + ...(body ? {body: JSON.stringify(body)} : {}), + }); + if (!res.ok) { + throw new Error(`${method} ${path} → ${res.status} ${await res.text()}`); + } + return res.status === 204 ? null : res.json(); +} + +async function mintOidcToken(audience) { + const url = process.env.ACTIONS_ID_TOKEN_REQUEST_URL; + const bearer = process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN; + if (!url || !bearer) { + return null; + } + const sep = url.includes('?') ? '&' : '?'; + const res = await fetch(`${url}${sep}audience=${encodeURIComponent(audience)}`, { + headers: {Authorization: `bearer ${bearer}`, Accept: 'application/json; api-version=2.0'}, + }); + if (!res.ok) { + throw new Error(`OIDC mint failed: ${res.status}`); + } + return (await res.json()).value || null; +} + +/** + * Record the correction against every verdict triage produced for this PR. + * + * All of them, not just the run-level one: the maintainer is correcting the + * conclusion, and leaving per-cluster rows uncorrected would leave the accuracy + * query reporting those clusters as unchallenged. + */ +async function recordCorrections({tsioUrl, credential, repo, prNumber, parsed, actor}) { + const headers = credential.apiKey ? + {'X-API-Key': credential.apiKey} : + {Authorization: `Bearer ${credential.token}`}; + + const listUrl = `${tsioUrl}/api/v1/triage/verdicts?repo=${encodeURIComponent(repo)}&pr=${prNumber}&limit=200`; + + // Credentials on the read too. It is a public endpoint today, so this is not + // required — but the read and the writes that follow it are one operation, + // and leaving the read anonymous means putting the endpoint behind auth later + // breaks override rather than being a no-op. + const listRes = await fetch(listUrl, {headers}); + if (!listRes.ok) { + throw new Error(`could not list verdicts: ${listRes.status}`); + } + const {verdicts} = await listRes.json(); + if (!verdicts || verdicts.length === 0) { + return {ok: false, corrected: 0, total: 0, note: 'no recorded verdicts for this PR'}; + } + + // Only the newest run's verdicts: older ones describe commits that are no + // longer what the checks reflect. Newest is resolved from created_at rather + // than by trusting the response order — the endpoint happens to sort + // newest-first, but correcting the wrong commit's verdicts is silent and + // permanent, which is too much to stake on an ordering nobody promised. + const newest = verdicts.reduce((a, b) => + (new Date(b.created_at) > new Date(a.created_at) ? b : a)); + const targets = verdicts.filter((v) => v.commit_sha === newest.commit_sha); + + let corrected = 0; + for (const v of targets) { + const res = await fetch(`${tsioUrl}/api/v1/triage/verdicts/${v.id}/correction`, { + method: 'POST', + headers: {...headers, 'Content-Type': 'application/json'}, + // corrected_by is not sent: TSIO derives attribution from the + // authenticated principal, because a body-supplied name could be + // anyone's. The maintainer is named in the PR comment below, under + // GitHub's own authentication. + body: JSON.stringify({ + corrected_verdict: parsed.verdict, + corrected_reason: parsed.reason, + }), + }); + if (res.ok) { + corrected += 1; + } else { + console.error(`correction for ${v.id} failed: ${res.status} ${await res.text()}`); + } + } + // ok is what the caller reports on, rather than the shape of the note. A run + // where every correction POST failed still produces "0/5 verdict(s) + // corrected", which reads as success to anything matching on that phrasing — + // and claiming a correction was recorded when none was is a false claim of + // accountability in the one place accountability is the product. + return {ok: corrected > 0, corrected, total: targets.length, commit: newest.commit_sha}; +} + +function arg(name, dflt = '') { + const hit = process.argv.slice(2).find((a) => a.startsWith(`--${name}=`)); + return hit === undefined ? dflt : hit.slice(name.length + 3); +} + +async function main() { + const repo = arg('repo'); + const prNumber = Number(arg('pr')); + const actor = arg('actor'); + const commentId = arg('comment-id'); + const tsioUrl = arg('tsio-url', 'https://test-io.test.mattermost.com'); + const statusContext = arg('status-context', DEFAULT_STATUS_CONTEXT); + const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN; + const body = process.env.COMMENT_BODY || ''; + + if (!token) { + throw new Error('GH_TOKEN is required'); + } + + const parsed = parseCommand(body); + if (!parsed.ok) { + // A malformed command gets a thumbs-down and an explanation rather than a + // silent no-op: the maintainer believes they have corrected something. + if (commentId) { + await gh(token, 'POST', `/repos/${repo}/issues/comments/${commentId}/reactions`, {content: 'confused'}); + } + await gh(token, 'POST', `/repos/${repo}/issues/${prNumber}/comments`, { + body: `:warning: **Triage override not applied** — ${parsed.error}`, + }); + console.log(`rejected: ${parsed.error}`); + return; + } + + const decision = decideAfterOverride(parsed); + console.log(JSON.stringify({parsed, decision})); + + // Resolve the head SHA now: the status has to land on the commit the checks + // are attached to, not on whatever the PR pointed at when triage ran. + const pr = await gh(token, 'GET', `/repos/${repo}/pulls/${prNumber}`); + const headSha = pr.head.sha; + + // 1. Record first — this is the part that outlives the PR. + let ledgerNote = 'not recorded'; + let recordedCleanly = false; + try { + const apiKey = process.env.TSIO_API_KEY || ''; + const oidc = apiKey ? null : await mintOidcToken(arg('tsio-audience', 'mattermost-test-system-io')); + if (apiKey || oidc) { + const result = await recordCorrections({ + tsioUrl, + credential: {apiKey, token: oidc}, + repo, + prNumber, + parsed, + actor, + }); + ledgerNote = result.note || `${result.corrected}/${result.total} verdict(s) corrected`; + recordedCleanly = Boolean(result.ok); + } else { + ledgerNote = 'no TSIO credential available'; + } + } catch (err) { + ledgerNote = `ledger write failed: ${err.message}`; + console.error(ledgerNote); + } + + // 2. Bring the checks into line with the human's decision. A human waiver + // wears the human label, never the AI one — conflating them makes the + // false-green metric uncomputable. Withdrawing a waiver removes both, so a + // correction to a real bug clears whichever label was carrying the green. + await gh(token, 'POST', `/repos/${repo}/statuses/${headSha}`, { + state: decision.state, + context: statusContext, + description: clampDescription(decision.description), + target_url: arg('run-url', ''), + }); + + try { + if (decision.applyLabel) { + await gh(token, 'POST', `/repos/${repo}/issues/${prNumber}/labels`, + {labels: [HUMAN_OVERRIDE_LABEL]}); + } else { + for (const label of [AI_WAIVED_LABEL, HUMAN_OVERRIDE_LABEL]) { + await gh(token, 'DELETE', + `/repos/${repo}/issues/${prNumber}/labels/${encodeURIComponent(label)}`); + } + } + } catch (err) { + if (decision.applyLabel || !/→ 404/.test(err.message)) { + console.error(`label update failed: ${err.message}`); + } + } + + if (commentId) { + await gh(token, 'POST', `/repos/${repo}/issues/comments/${commentId}/reactions`, {content: '+1'}); + } + + await gh(token, 'POST', `/repos/${repo}/issues/${prNumber}/comments`, { + body: [ + `:white_check_mark: **Triage override applied by @${actor}**`, + '', + `\`${statusContext}\` is now **${decision.state}** — \`${parsed.verdict}\`: ${parsed.reason}`, + '', + recordedCleanly ? + `_Correction recorded (${ledgerNote}). It counts toward the triage accuracy metrics._` : + `:warning: _The check was updated, but the correction was **not** recorded: ${ledgerNote}. ` + + 'The accuracy metrics will not see this one._', + ].join('\n'), + }); + + console.log(`override applied: ${decision.state} (${ledgerNote})`); +} + +if (require.main === module) { + main().catch((err) => { + console.error(`triage-override failed: ${err.stack || err.message}`); + process.exit(1); + }); +} + +module.exports = { + parseCommand, + decideAfterOverride, + clampDescription, + STATUS_DESCRIPTION_LIMIT, + VERDICTS, + WAIVABLE, + AI_WAIVED_LABEL, + HUMAN_OVERRIDE_LABEL, + DEFAULT_STATUS_CONTEXT, +}; diff --git a/scripts/triage-override.test.js b/scripts/triage-override.test.js new file mode 100644 index 0000000..564bfbb --- /dev/null +++ b/scripts/triage-override.test.js @@ -0,0 +1,133 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +const assert = require('node:assert/strict'); +const {test} = require('node:test'); + +const { + STATUS_DESCRIPTION_LIMIT, + clampDescription, + decideAfterOverride, + parseCommand, + AI_WAIVED_LABEL, + HUMAN_OVERRIDE_LABEL, +} = require('./triage-override'); + +// ---------- parsing ---------- + +test('a well-formed command parses', () => { + const p = parseCommand('/e2e-triage-override PR_REGRESSION this really was broken by the change'); + + assert.equal(p.ok, true); + assert.equal(p.verdict, 'PR_REGRESSION'); + assert.equal(p.reason, 'this really was broken by the change'); + assert.equal(p.waivable, false); +}); + +test('verdicts are accepted in the form people actually type them', () => { + for (const input of ['flaky-infra', 'FLAKY_INFRA', 'Flaky-Infra', 'flaky_infra']) { + const p = parseCommand(`/e2e-triage-override ${input} emulator died again`); + + assert.equal(p.ok, true, `${input} should parse`); + assert.equal(p.verdict, 'FLAKY_INFRA'); + } +}); + +test('the command is found on any line of a longer comment', () => { + const p = parseCommand('I looked into this.\n\n/e2e-triage-override TEST_DEBT selector went stale\n\nthanks'); + + assert.equal(p.ok, true); + assert.equal(p.verdict, 'TEST_DEBT'); + assert.equal(p.reason, 'selector went stale'); +}); + +test('a reason is mandatory', () => { + const p = parseCommand('/e2e-triage-override FLAKY_TEST'); + + assert.equal(p.ok, false); + assert.match(p.error, /reason is required/); +}); + +test('an unknown verdict is rejected with the valid list', () => { + const p = parseCommand('/e2e-triage-override NOT_MY_PROBLEM it is fine honestly'); + + assert.equal(p.ok, false); + assert.match(p.error, /not a known verdict/); + assert.match(p.error, /PR_REGRESSION/); +}); + +test('a bare command explains the usage', () => { + const p = parseCommand('/e2e-triage-override'); + + assert.equal(p.ok, false); + assert.match(p.error, /usage/); +}); + +test('an unrelated comment is not a command', () => { + assert.equal(parseCommand('looks flaky to me').ok, false); + assert.equal(parseCommand('').ok, false); +}); + +// ---------- resulting check state ---------- + +test('correcting to a not-your-fault verdict greens the check and applies the label', () => { + const d = decideAfterOverride(parseCommand('/e2e-triage-override FLAKY_INFRA runner lost adb')); + + assert.equal(d.state, 'success'); + assert.equal(d.applyLabel, true); + assert.match(d.description, /human override/); +}); + +test('correcting to a real-bug verdict reds the check and withdraws the waiver', () => { + const d = decideAfterOverride(parseCommand('/e2e-triage-override PR_REGRESSION the change broke it')); + + assert.equal(d.state, 'failure'); + assert.equal( + d.applyLabel, false, + 'the label is sticky and would keep greening later commits if left applied', + ); +}); + +test('INCONCLUSIVE is treated as unresolved, so it reds', () => { + const d = decideAfterOverride(parseCommand('/e2e-triage-override INCONCLUSIVE nobody knows yet')); + + assert.equal(d.state, 'failure'); + assert.equal(d.applyLabel, false); +}); + +test('the description carries the verdict and the human reason in full', () => { + // Unsliced on purpose. The previous assertion checked + // `d.description.slice(0, 140).length <= 140`, which is true of every string + // and so asserted nothing — and it checked truncation on the one function + // that deliberately does not truncate. decideAfterOverride returns the whole + // reason because the PR comment prints all of it; only the commit status is + // capped, and that happens at the status call. + const reason = 'x'.repeat(400); + const d = decideAfterOverride(parseCommand(`/e2e-triage-override FLAKY_TEST ${reason}`)); + + assert.match(d.description, /flaky-test/); + assert.ok(d.description.includes(reason), 'the maintainer reason must survive intact'); +}); + +// ---------- AI vs human overrides stay distinguishable ---------- + +test('a human waiver wears a distinct label from an AI waiver', () => { + assert.notEqual(AI_WAIVED_LABEL, HUMAN_OVERRIDE_LABEL, + 'conflating them makes the false-green metric uncomputable'); + assert.equal(AI_WAIVED_LABEL, 'E2E/AI-Waived'); + assert.equal(HUMAN_OVERRIDE_LABEL, 'E2E/Override'); +}); + +test('the commit-status description is capped at the GitHub limit', () => { + // The cap GitHub enforces silently. Asserting it here rather than at the + // network call keeps it a tested rule instead of a bare slice. + const long = decideAfterOverride(parseCommand(`/e2e-triage-override FLAKY_TEST ${'x'.repeat(400)}`)); + const clamped = clampDescription(long.description); + + assert.equal(clamped.length, STATUS_DESCRIPTION_LIMIT); + assert.ok(clamped.startsWith('human override: flaky-test'), 'the verdict must survive truncation'); + + // Short descriptions pass through untouched. + const short = decideAfterOverride(parseCommand('/e2e-triage-override FLAKY_TEST it flakes')); + assert.equal(clampDescription(short.description), short.description); +});