Skip to content

refactor(cli): cover experimental and small families with effect lint (CLI-2412) #383

refactor(cli): cover experimental and small families with effect lint (CLI-2412)

refactor(cli): cover experimental and small families with effect lint (CLI-2412) #383

Workflow file for this run

name: AI Review
# One-shot AI code review: replaces the Codex GitHub App's automatic per-push reviews with a
# single exhaustive pass that runs at most once per PR. See .github/ai-review/README.md for the
# full design and security model.
#
# Triggered by workflow_dispatch (testing/ad-hoc), an internal maintainer commenting
# `/ai-review`, or automatically when a PR opens or leaves draft — resolve.ts gates the automatic
# path to PR authors with write access; external contributors go through the manual path instead.
on:
workflow_dispatch:
inputs:
pr:
description: "PR number to review"
required: true
type: string
issue_comment:
types:
- created
pull_request:
types:
- opened
- ready_for_review
permissions: {}
# One source of truth for the model names; resolve, claude-review, codex-review, and
# post-review's footer all read these instead of hardcoding them separately.
env:
CLAUDE_MODEL: claude-opus-5
CODEX_MODEL: gpt-5.6-sol
# Non-command issue_comment events fire for every comment on every PR, so grouping by PR number
# alone would let any comment cancel an in-flight review; give those runs their own per-run
# group instead. The command check is exact equality (mirroring resolve.ts), so a near-miss like
# `/ai-reviewers` can't land in the shared group and cancel a real review.
#
# `pull_request` events get a separate per-PR `auto` group from the manual `review` group, since
# an auto event may resolve to a skip and would otherwise cancel an in-flight maintainer-requested
# review without replacing it. An auto and manual run can overlap on the same PR as a result —
# rare, and self-healing since the later post supersedes the earlier one.
concurrency:
group: >-
ai-review-${{ github.event.pull_request.number || github.event.issue.number || inputs.pr }}-${{
(github.event_name == 'issue_comment' && github.event.comment.body != '/ai-review')
&& github.run_id
|| (github.event_name == 'pull_request' && 'auto' || 'review') }}
cancel-in-progress: true
jobs:
resolve:
name: Resolve
runs-on: ubuntu-latest
timeout-minutes: 5
# Only PR comments starting with /ai-review, from an association that could plausibly be a
# maintainer, reach this job — a cheap, non-authoritative pre-filter (it can't see a private
# org member's real permission, so it can under-admit). The authoritative checks — exact
# command match and effective-permission lookup — happen in resolve.ts.
if: >
github.event_name != 'issue_comment' ||
(github.event.issue.pull_request != null &&
startsWith(github.event.comment.body, '/ai-review') &&
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association))
# Reacts 👀 to the triggering comment (pull-requests: write) but runs only trusted,
# default-branch code (see the pinned checkout ref below), never a PR's own — safe to grant write.
permissions:
pull-requests: write
contents: read
outputs:
should_run: ${{ steps.resolve.outputs.should_run }}
pr_number: ${{ steps.resolve.outputs.pr_number }}
head_ref: ${{ steps.resolve.outputs.head_ref }}
trigger: ${{ steps.resolve.outputs.trigger }}
steps:
# Pinned to the default ref explicitly — this job must keep running only trusted repo code,
# even though the pull_request trigger above hands it PR-authored event payloads.
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.repository.default_branch }}
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version-file: ".bun-version"
- name: Resolve
id: resolve
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_REPOSITORY: ${{ github.repository }}
EVENT_NAME: ${{ github.event_name }}
PR_NUMBER: ${{ inputs.pr || github.event.issue.number || github.event.pull_request.number }}
COMMENT_ID: ${{ github.event.comment.id }}
COMMENT_AUTHOR_LOGIN: ${{ github.event.comment.user.login }}
COMMENT_AUTHOR_ASSOCIATION: ${{ github.event.comment.author_association }}
COMMENT_BODY: ${{ github.event.comment.body }}
run: bun .github/scripts/ai-review/resolve.ts
claude-review:
name: Claude review
needs: resolve
if: needs.resolve.outputs.should_run == 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
# Security-critical: this job checks out the PR's own head commit as untrusted review subject
# matter only — nothing it executes may come from that checkout. Prompts, the findings schema,
# and the validation script are all read from a separate trusted default-branch checkout
# (`path: trusted` below). The job holds no write permissions, a read-only Claude tool
# allowlist (no write/edit tools, no Bash), and no secrets beyond ANTHROPIC_API_KEY.
permissions:
contents: read
pull-requests: read
steps:
- name: Checkout PR head (untrusted; review subject matter only)
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ needs.resolve.outputs.head_ref }}
path: pr
fetch-depth: 1
persist-credentials: false
- name: Checkout default branch (trusted; everything we execute comes from here)
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.repository.default_branch }}
path: trusted
fetch-depth: 0
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
# The PR head's own `.bun-version` is untrusted — it could select a
# canary/malicious toolchain — so read it from the trusted checkout.
bun-version-file: "trusted/.bun-version"
# This run's cache scope is the default branch; an untrusted run
# must never be able to write to it.
no-cache: true
- name: Generate PR diff and fetch metadata
working-directory: trusted
env:
GH_TOKEN: ${{ github.token }}
PR: ${{ needs.resolve.outputs.pr_number }}
run: |
mkdir -p /tmp/ai-review
gh pr view "$PR" --repo "$GITHUB_REPOSITORY" \
--json number,title,body,baseRefName,headRefName,additions,deletions,changedFiles \
> /tmp/ai-review/pr.json
base_ref=$(jq -r '.baseRefName' /tmp/ai-review/pr.json)
bun .github/scripts/ai-review/generate-pr-diff.ts "$PR" "$base_ref"
# Pin the exact published version so a Claude Code release can't silently change review
# behavior mid-rollout. Installed from the trusted checkout with npm config isolation so a
# PR-supplied `.npmrc` in the untrusted `pr` checkout can never redirect this install to a
# hostile registry.
- name: Install Claude Code CLI
working-directory: trusted
run: |
# Isolate npm config with two DISTINCT empty paths — npm rejects the
# same path for --userconfig and --globalconfig ("double-loading
# config '/dev/null'"). These paths don't exist, so npm uses empty
# user/global config; running from `trusted/` already avoids the
# untrusted `pr` checkout's project `.npmrc`.
npm install -g \
--userconfig "${RUNNER_TEMP}/ai-review-npmrc-user" \
--globalconfig "${RUNNER_TEMP}/ai-review-npmrc-global" \
--registry=https://registry.npmjs.org/ @anthropic-ai/claude-code@2.1.247
# Security-critical invariant: PR code is only ever read by `claude`, in the `pr`-cwd
# subshell below — no other command in this job, including any `bun` call, runs with a cwd
# inside `pr`. `bun` auto-loads `bunfig.toml` (arbitrary `preload` code) and `.env` from its
# cwd, so a `pr`-cwd `bun` call would let a PR-authored `bunfig.toml` execute in a step
# holding `ANTHROPIC_API_KEY`. `claude` is a standalone binary unaffected by `bunfig.toml`;
# `--bare` disables hooks/MCP/CLAUDE.md, and `--strict-mcp-config` guards against a future
# CLI regression.
- name: Run Claude review
working-directory: trusted
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
run: |
# GitHub launches this with `bash -e`; the retry loop below inspects
# exit codes manually (a non-zero `claude` is expected and retried),
# so errexit must be OFF — otherwise the failing subshell aborts the
# step before cli_exit/is_error are checked and the retry never runs.
set +e -uo pipefail
success=false
for attempt in 1 2; do
(
cd "$GITHUB_WORKSPACE/pr" &&
claude --bare --strict-mcp-config -p "$(cat "$GITHUB_WORKSPACE/trusted/.github/ai-review/claude-review-prompt.md")" \
--model "$CLAUDE_MODEL" \
--output-format json \
--json-schema "$(jq -c 'del(.["$schema"])' "$GITHUB_WORKSPACE/trusted/.github/ai-review/findings.schema.json")" \
--allowedTools "Read,Grep,Glob" \
--max-turns 200
) > /tmp/ai-review/claude-raw.json
cli_exit=$?
# `--json-schema` makes the CLI populate `.structured_output` on a
# genuine success; it stays null on a hard failure such as
# `error_max_turns` — a truncated max-turns response shouldn't be
# trusted just because some text happens to end up in `.result`,
# so there's no `.result`-parsing fallback here.
is_error="true"
structured_output_is_null="true"
if [ "$cli_exit" -eq 0 ]; then
is_error=$(jq -r '.is_error == true' /tmp/ai-review/claude-raw.json 2>/dev/null || echo "true")
structured_output_is_null=$(jq -r '.structured_output == null' /tmp/ai-review/claude-raw.json 2>/dev/null || echo "true")
fi
if [ "$cli_exit" -eq 0 ] && [ "$is_error" = "false" ] && [ "$structured_output_is_null" = "false" ] &&
jq -c '.structured_output' /tmp/ai-review/claude-raw.json > /tmp/ai-review/claude-findings.json 2>/dev/null &&
bun .github/scripts/ai-review/post-review.ts validate-findings /tmp/ai-review/claude-findings.json
then
success=true
break
fi
echo "Claude review attempt $attempt failed (cli_exit=$cli_exit, is_error=$is_error, structured_output_null=$structured_output_is_null); retrying..." >&2
done
if [ "$success" != "true" ]; then
echo "::error ::Claude review failed after 2 attempts." >&2
exit 1
fi
# Scrubs any secret-shaped substring a prompt-injected model might have echoed back before
# the raw JSON is uploaded as a public-repo artifact; the posted review is scrubbed
# separately at render time. Runs with `if: always()` so a partial `claude-raw.json` from a
# failed attempt is still scrubbed before the always-on upload step.
- name: Redact secrets from Claude findings
if: always()
working-directory: trusted
run: |
for f in /tmp/ai-review/claude-findings.json /tmp/ai-review/claude-raw.json; do
if [ -f "$f" ]; then
# Delete the file if redaction fails, so the always-on upload
# below can never publish an unscrubbed artifact.
bun .github/scripts/ai-review/post-review.ts redact "$f" || { rm -f "$f"; exit 1; }
fi
done
- name: Upload Claude findings
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: claude-findings
path: |
/tmp/ai-review/claude-findings.json
/tmp/ai-review/claude-raw.json
retention-days: 3
codex-review:
name: Codex review
needs: resolve
if: needs.resolve.outputs.should_run == 'true'
# Codex's independent review doesn't depend on claude-review, so it runs in parallel. It
# works purely from /tmp/ai-review/pr.diff, so it needs only the trusted default-branch
# checkout — no PR-head checkout. The verify-by-reading step (which needs the PR's files) is
# the separate `adjudicate` job below.
permissions:
contents: read
pull-requests: read
timeout-minutes: 45
runs-on: ubuntu-latest
steps:
- name: Checkout default branch (trusted; the only checkout this job needs)
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.repository.default_branch }}
fetch-depth: 0
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version-file: ".bun-version"
no-cache: true
- name: Generate PR diff
env:
GH_TOKEN: ${{ github.token }}
PR: ${{ needs.resolve.outputs.pr_number }}
run: |
mkdir -p /tmp/ai-review
base_ref=$(gh pr view "$PR" --repo "$GITHUB_REPOSITORY" --json baseRefName --jq '.baseRefName')
bun .github/scripts/ai-review/generate-pr-diff.ts "$PR" "$base_ref"
- name: Prepare findings output schema
run: |
mkdir -p /tmp/ai-review
jq 'del(.["$schema"])' .github/ai-review/findings.schema.json > /tmp/ai-review/findings.schema.json
# Safety strategy (drop-sudo + read-only) — see the "Run Codex adjudication" step below for
# the full rationale (verified against the pinned openai/codex-action's action.yml +
# src/runCodexExec.ts). In short: non-sudo user, no filesystem writes, no network.
- name: Run Codex independent review
# Pinned to v1.11 (52fe01ec…), not v1.12 (86365089…): v1.12 has two open upstream
# regressions on this workflow's config (safety-strategy: drop-sudo, sandbox: read-only,
# output-schema-file) — openai/codex-action#151 (a lingering descendant process keeps the
# step alive after Codex already wrote its output) and openai/codex-action#160
# (drop-sudo's rewrite chmods root-owned /run sockets, breaking systemd-resolved and
# killing the job regardless of timeout-minutes). A dependabot.yml ignore entry for
# openai/codex-action stops an automated bump from silently regressing this pin;
# re-verify both issues are closed before ever re-bumping it.
uses: &codex-action-pin openai/codex-action@52fe01ec70a42f454c9d2ebd47598f9fd6893d56 # v1.11
with:
openai-api-key: ${{ secrets.OPENAI_API_KEY }}
prompt-file: .github/ai-review/codex-review-prompt.md
model: ${{ env.CODEX_MODEL }}
effort: high
output-schema-file: /tmp/ai-review/findings.schema.json
output-file: /tmp/ai-review/codex-findings.json
codex-version: "0.150.1"
working-directory: ${{ github.workspace }}
safety-strategy: drop-sudo
sandbox: read-only
- name: Validate Codex findings
run: bun .github/scripts/ai-review/post-review.ts validate-findings /tmp/ai-review/codex-findings.json
# Same defense-in-depth as claude-review's redact step: scrub any
# secret-shaped substring out of the findings before they're uploaded as
# a (public-repo) artifact.
- name: Redact secrets from Codex findings
if: always()
run: |
if [ -f /tmp/ai-review/codex-findings.json ]; then
# Delete on redaction failure so the always-on upload can't publish
# an unscrubbed artifact.
bun .github/scripts/ai-review/post-review.ts redact /tmp/ai-review/codex-findings.json \
|| { rm -f /tmp/ai-review/codex-findings.json; exit 1; }
fi
- name: Upload Codex findings
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: codex-findings
path: /tmp/ai-review/codex-findings.json
retention-days: 3
adjudicate:
name: Adjudicate reviews
needs:
- resolve
- claude-review
- codex-review
# Runs when at least one independent review succeeded — a single flaky model job must not
# sink the whole review. Findings download steps are guarded per job result, and staging
# substitutes an empty findings set for any review that didn't complete.
if: ${{ !cancelled() && needs.resolve.outputs.should_run == 'true' && (needs.claude-review.result == 'success' || needs.codex-review.result == 'success') }}
# Security-critical: this job checks out the PR head (untrusted subject matter) so Codex can
# verify findings by reading the real files. Codex's working directory is the workspace root,
# which holds only `pr/` and `trusted/` (no AGENTS.md/config of its own); it reads `pr/`
# read-only, and the adjudicate prompt's injection guard treats every file under `pr/`
# (including any AGENTS.md/CLAUDE.md) as untrusted data. Every `bun` invocation runs from
# `trusted/`. A prompt-injected Codex here is bounded to review content: read-only sandbox,
# no network, key proxied by the action, and secret-scrubbed output.
permissions:
contents: read
pull-requests: read
timeout-minutes: 45
runs-on: ubuntu-latest
steps:
- name: Checkout PR head (untrusted; read-only, for verify-by-reading)
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ needs.resolve.outputs.head_ref }}
path: pr
fetch-depth: 1
persist-credentials: false
- name: Checkout default branch (trusted; everything we execute comes from here)
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.repository.default_branch }}
path: trusted
fetch-depth: 0
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
# The PR head's own `.bun-version` is untrusted; read it from trusted.
bun-version-file: "trusted/.bun-version"
no-cache: true
- name: Download Claude findings
if: needs.claude-review.result == 'success'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: claude-findings
path: ${{ runner.temp }}/claude-in
- name: Download Codex findings
if: needs.codex-review.result == 'success'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: codex-findings
path: ${{ runner.temp }}/codex-in
- name: Stage findings
run: |
mkdir -p /tmp/ai-review
# Copy only the expected filenames rather than trusting the zips' own
# entry paths (artifacts are, in principle, upstream-influenced). If a
# review job didn't complete, substitute an empty findings set so the
# adjudicator always has both files and simply reconciles the one that
# did run.
claude_src="${{ runner.temp }}/claude-in/claude-findings.json"
codex_src="${{ runner.temp }}/codex-in/codex-findings.json"
if [ -f "$claude_src" ]; then
cp "$claude_src" /tmp/ai-review/claude-findings.json
else
echo '{"summary":"Claude review did not complete for this run.","findings":[]}' \
> /tmp/ai-review/claude-findings.json
fi
if [ -f "$codex_src" ]; then
cp "$codex_src" /tmp/ai-review/codex-findings.json
else
echo '{"summary":"Codex review did not complete for this run.","findings":[]}' \
> /tmp/ai-review/codex-findings.json
fi
- name: Generate PR diff
working-directory: trusted
env:
GH_TOKEN: ${{ github.token }}
PR: ${{ needs.resolve.outputs.pr_number }}
run: |
mkdir -p /tmp/ai-review
base_ref=$(gh pr view "$PR" --repo "$GITHUB_REPOSITORY" --json baseRefName --jq '.baseRefName')
bun .github/scripts/ai-review/generate-pr-diff.ts "$PR" "$base_ref"
- name: Prepare merged-review output schema
working-directory: trusted
run: |
mkdir -p /tmp/ai-review
jq 'del(.["$schema"])' .github/ai-review/merged-review.schema.json > /tmp/ai-review/merged-review.schema.json
# Safety strategy, per the pinned openai/codex-action's action.yml + src/runCodexExec.ts:
# - `safety-strategy: read-only` forces the legacy sandbox read-only, but Codex still
# runs as the action's default sudo-capable user — the action's own docs call this
# unsafe, since a sudo-capable process can read secrets like OPENAI_API_KEY out of
# memory even under a read-only, no-network sandbox.
# - `safety-strategy: drop-sudo` (the action's default) removes sudo from the user
# running Codex but says nothing on its own about the filesystem/network sandbox.
# - The action only forces its legacy read-only sandbox when `safety-strategy ===
# "read-only"`; otherwise it honors a separately-set `sandbox` input as-is. Setting
# `safety-strategy: drop-sudo` and `sandbox: read-only` together composes them safely:
# non-sudo user, no filesystem writes, no network.
# `working-directory` is the workspace root so Codex's cwd holds no untrusted AGENTS.md/
# config; it reads the PR from `pr/` and executes nothing from it.
- name: Run Codex adjudication
# Same v1.11 pin as codex-review's step above (see its comment for the full rationale);
# YAML-aliased so the SHA only needs to change in one place.
uses: *codex-action-pin
with:
openai-api-key: ${{ secrets.OPENAI_API_KEY }}
prompt-file: trusted/.github/ai-review/adjudicate-prompt.md
model: ${{ env.CODEX_MODEL }}
effort: high
output-schema-file: /tmp/ai-review/merged-review.schema.json
output-file: /tmp/ai-review/merged-review.json
codex-version: "0.150.1"
working-directory: ${{ github.workspace }}
safety-strategy: drop-sudo
sandbox: read-only
- name: Validate merged review
working-directory: trusted
run: bun .github/scripts/ai-review/post-review.ts validate-merged /tmp/ai-review/merged-review.json
- name: Redact secrets from merged review
if: always()
working-directory: trusted
run: |
if [ -f /tmp/ai-review/merged-review.json ]; then
# Delete on redaction failure so the always-on upload can't publish
# an unscrubbed artifact.
bun .github/scripts/ai-review/post-review.ts redact /tmp/ai-review/merged-review.json \
|| { rm -f /tmp/ai-review/merged-review.json; exit 1; }
fi
- name: Upload merged review
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: merged-review
path: /tmp/ai-review/merged-review.json
retention-days: 3
post-review:
name: Post review
needs:
- resolve
- adjudicate
# Runs only when adjudication succeeded (it produced the merged review this job posts).
# `!cancelled()` is required since an explicit `if` replaces the default "all needed jobs
# succeeded" check.
if: ${{ !cancelled() && needs.resolve.outputs.should_run == 'true' && needs.adjudicate.result == 'success' }}
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
pull-requests: write
steps:
# Security-critical: this is the only job with write permission, so it must only ever
# execute trusted base-branch code — never the PR head. Checking out `develop` explicitly
# (never `needs.resolve.outputs.head_ref`) keeps a malicious PR from smuggling a script
# change into the one job that can write back to the PR. (For `pull_request` events GitHub
# runs the workflow file from the PR's own ref; acceptable since the auto path only admits
# same-repo PRs, whose authors hold write access anyway, and fork PRs run with a read-only
# token and no secrets.)
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: develop
persist-credentials: false
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version-file: ".bun-version"
- name: Download merged review
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: merged-review
path: /tmp/ai-review
- name: Post review
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_REPOSITORY: ${{ github.repository }}
PR_NUMBER: ${{ needs.resolve.outputs.pr_number }}
MERGED_REVIEW_PATH: /tmp/ai-review/merged-review.json
TRIGGER: ${{ needs.resolve.outputs.trigger }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
# CLAUDE_MODEL / CODEX_MODEL are inherited from the workflow-level `env:` block above,
# so the footer never drifts from what actually ran.
run: bun .github/scripts/ai-review/post-review.ts post