Add reusable E2E AI failure-triage adjudication workflow

.github/workflows/ci.yml

@@ -0,0 +1,66 @@
+---
+# CI for the toolkit's own scripts.
+#
+# The policy engine in scripts/triage-policy.js decides whether an E2E failure
+# blocks a merge across every repo that calls the reusable workflow. Shipping it
+# without its tests running would mean a regression in "when is a red allowed to
+# turn green" reaches consumers silently — which is the one failure mode the
+# whole design is built to prevent.
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Script tests
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
+        with:
+          node-version: '22'
+
+      - name: ci/test
+        # Files are listed explicitly rather than passing the directory: node's
+        # directory runner also executes non-test sources, which turns a plain
+        # `require` into a spurious failure.
+        run: |
+          node --test \
+            scripts/triage-policy.test.js \
+            scripts/triage-apply.test.js \
+            scripts/triage-override.test.js \
+            scripts/triage-blame.test.js
+
+  actionlint:
+    name: Workflow lint
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: ci/actionlint
+        run: |
+          set -euo pipefail
+          # Pinned by commit, not by branch. `.../main/scripts/...` re-fetches
+          # whatever that branch holds at the moment CI runs, so this step used to
+          # pipe a mutable remote script straight into bash on a runner holding
+          # the workflow token. The commit ref is immutable, and the version the
+          # script then downloads is fixed rather than "latest".
+          INSTALLER_REF=3795ba2f6cb243eeca54c9d22e5c531cb9dcfb4a
+          ACTIONLINT_VERSION=1.7.12
+          curl -fsSL -o download-actionlint.bash \
+            "https://raw.githubusercontent.com/rhysd/actionlint/${INSTALLER_REF}/scripts/download-actionlint.bash"
+          bash download-actionlint.bash "${ACTIONLINT_VERSION}"
+          # shellcheck is not installed on the runner image by default; the
+          # workflow-level checks are what matter here.
+          ./actionlint -shellcheck= .github/workflows/*.yml

.github/workflows/e2e-ai-triage-override.yml

@@ -0,0 +1,201 @@
+---
+# Human override of an automated triage verdict.
+#
+# Corrections are the only ground truth this system gets — everything else is the
+# triage grading its own homework. A maintainer saying "that was actually a real
+# bug" is the one input that can prove a waiver wrong, and it is exactly what the
+# false-green metric counts.
+#
+# The correction is recorded in the TSIO ledger *and* the checks are brought into
+# line with what the human said. If the ledger write fails the checks are still
+# updated — the maintainer's intent must be honoured — but the reply says so,
+# because an unrecorded correction is a data point permanently lost.
+name: E2E AI Triage Override (Reusable)
+
+on:
+  workflow_call:
+    inputs:
+      target_repo:
+        description: "Full repo name the PR and statuses belong to"
+        required: true
+        type: string
+      pr_number:
+        required: true
+        type: string
+      comment_body:
+        description: "Raw comment body, parsed for /e2e-triage-override <verdict> <reason>"
+        required: true
+        type: string
+      comment_id:
+        description: "Comment to react to, so the author sees it was picked up"
+        required: false
+        type: string
+        default: ""
+      sender:
+        description: "Who issued the override — recorded as corrected_by"
+        required: true
+        type: string
+      tsio_url:
+        required: false
+        type: string
+        default: "https://test-io.test.mattermost.com"
+      toolkit_ref:
+        description: >-
+          Ref of THIS repository to check out for the scripts. Defaults to main.
+          It cannot be derived: inside a called reusable workflow the `github`
+          context — including workflow_ref and workflow_sha — describes the
+          *caller*, so resolving from it would try to check this repo out at the
+          caller's branch. Callers testing an unmerged toolkit change must pass
+          the same ref they pinned `uses:` to.
+        required: false
+        type: string
+        default: "main"
+    secrets:
+      GH_TOKEN:
+        description: "Token for statuses, labels, and comments on target_repo"
+        required: true
+      TSIO_API_KEY:
+        description: "Optional. Without it the ledger write uses a minted OIDC token."
+        required: false
+
+permissions:
+  contents: read
+  # The ledger write authenticates with a minted OIDC token by default.
+  id-token: write
+
+jobs:
+  override:
+    runs-on: ubuntu-24.04
+    steps:
+      # Check out THIS repository, not the caller's.
+      #
+      # In a reusable workflow `github.repository` is the *caller*, so a bare
+      # checkout cloned mattermost-mobile and the next step ran
+      # `node scripts/...` against a tree with no such file. MODULE_NOT_FOUND
+      # kills the job before the red fallback can post, so the run ends with no
+      # status at all — and absent is not fail-closed, it is silence.
+      #
+      # The ref comes from an input rather than from github.workflow_ref: that
+      # context also describes the caller, so deriving from it would ask for this
+      # repository at the caller's branch, which does not exist here.
+      - name: ci/checkout-toolkit
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          repository: mattermost/mattermost-test-automation-toolkit
+          ref: ${{ inputs.toolkit_ref }}
+          persist-credentials: false
+
+      - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0
+        with:
+          node-version: '22'
+
+      # Verify the sender here rather than trusting the caller to have done it.
+      #
+      # `sender` is just a string input: this workflow is reusable, so whether it
+      # was gated upstream is a property of each caller, not of this file. A
+      # caller that forgets — or one that passes github.actor from a trigger that
+      # any user can fire — would let an outside contributor overturn a red E2E
+      # verdict and apply E2E/AI-Waived to their own PR. The permission is
+      # therefore checked against the repository the override targets, before
+      # anything is written.
+      - name: ci/verify-sender-can-write
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+          TARGET_REPO: ${{ inputs.target_repo }}
+          SENDER: ${{ inputs.sender }}
+          COMMENT_ID: ${{ inputs.comment_id }}
+          PR_NUMBER: ${{ inputs.pr_number }}
+        run: |
+          set -euo pipefail
+          if [ -z "${SENDER}" ]; then
+            echo "::error::sender is empty — refusing to apply an unattributed override"
+            exit 1
+          fi
+          PERM=$(gh api "repos/${TARGET_REPO}/collaborators/${SENDER}/permission" \
+            --jq '.permission' 2>/dev/null || echo "none")
+          echo "sender ${SENDER} has '${PERM}' on ${TARGET_REPO}"
+          case "${PERM}" in
+            admin|maintain|write) ;;
+            *)
+              echo "::error::${SENDER} needs write access to ${TARGET_REPO} to override a triage verdict (has '${PERM}')"
+              exit 1
+              ;;
+          esac
+
+          # Confirm the sender against the comment itself, rather than believing
+          # the caller. The permission check above already keeps outsiders out,
+          # but sender is still a caller-supplied string, so one collaborator
+          # could be recorded as having overturned a verdict another collaborator
+          # actually overturned. Attribution that can be chosen by the caller is
+          # not attribution — the same reason corrected_by is taken from the
+          # authenticated principal rather than the request body.
+          if [ -z "${COMMENT_ID}" ]; then
+            echo "::error::comment_id is required — the override cannot be traced to a comment without it"
+            exit 1
+          fi
+          COMMENT=$(gh api "repos/${TARGET_REPO}/issues/comments/${COMMENT_ID}" 2>/dev/null || echo '')
+          if [ -z "${COMMENT}" ]; then
+            echo "::error::comment ${COMMENT_ID} not found on ${TARGET_REPO}"
+            exit 1
+          fi
+          COMMENT_AUTHOR=$(printf '%s' "${COMMENT}" | jq -r '.user.login // ""')
+          if [ "${COMMENT_AUTHOR}" != "${SENDER}" ]; then
+            echo "::error::comment ${COMMENT_ID} was written by '${COMMENT_AUTHOR}', not '${SENDER}'"
+            exit 1
+          fi
+          # And that the comment belongs to the PR being overridden, so a comment
+          # from an unrelated issue cannot authorise a change here.
+          COMMENT_ISSUE=$(printf '%s' "${COMMENT}" | jq -r '.issue_url // ""')
+          if [ "${COMMENT_ISSUE##*/}" != "${PR_NUMBER}" ]; then
+            echo "::error::comment ${COMMENT_ID} belongs to ${COMMENT_ISSUE##*/}, not PR ${PR_NUMBER}"
+            exit 1
+          fi
+          echo "verified: ${SENDER} wrote comment ${COMMENT_ID} on PR ${PR_NUMBER}"
+
+      # Validate the TSIO origin before any credential can be sent to it.
+      #
+      # tsio_url is a caller-supplied string, and the next step sends TSIO_API_KEY
+      # — or a minted OIDC token — to whatever host it names. Unvalidated, that is
+      # a credential-exfiltration primitive rather than a wrong endpoint: a caller
+      # passing https://attacker.example gets the ledger token posted to it. Exact
+      # origins only, no suffix matching, because "endswith mattermost.com" is
+      # satisfied by evil-mattermost.com.
+      - name: ci/validate-tsio-origin
+        env:
+          TSIO_URL: ${{ inputs.tsio_url }}
+        run: |
+          set -euo pipefail
+          case "${TSIO_URL}" in
+            https://test-io.test.mattermost.com|https://staging-test-io.test.mattermost.com) ;;
+            *)
+              echo "::error::tsio_url is not an approved TSIO origin: ${TSIO_URL}"
+              exit 1
+              ;;
+          esac
+          echo "TSIO origin approved: ${TSIO_URL}"
+
+      - name: ci/apply-override
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+          TSIO_API_KEY: ${{ secrets.TSIO_API_KEY }}
+          # Passed through the environment, never interpolated into the script
+          # arguments: a comment body is attacker-controlled text and must not be
+          # able to reach a shell as anything but data.
+          COMMENT_BODY: ${{ inputs.comment_body }}
+          # Same reasoning for the rest: sender is a GitHub login and the others
+          # are caller-supplied, so none of them are pasted into the script text.
+          TARGET_REPO: ${{ inputs.target_repo }}
+          PR_NUMBER: ${{ inputs.pr_number }}
+          SENDER: ${{ inputs.sender }}
+          COMMENT_ID: ${{ inputs.comment_id }}
+          TSIO_URL: ${{ inputs.tsio_url }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          node scripts/triage-override.js \
+            --repo="${TARGET_REPO}" \
+            --pr="${PR_NUMBER}" \
+            --actor="${SENDER}" \
+            --comment-id="${COMMENT_ID}" \
+            --tsio-url="${TSIO_URL}" \
+            --run-url="${RUN_URL}"