feat(evi): request native Slack approval for external PR reviews

apps/evi/.env.example

@@ -15,3 +15,9 @@
 # Use the token rather than EVE_MEMORY_BLOB_STORE_ID, whose OIDC authorization
 # only covers the environments the store is connected to.
 # EVE_MEMORY_BLOB_READ_WRITE_TOKEN=vercel_blob_rw_...
+
+# Configure a dedicated channel for external PR review approvals.
+# Invite Evi and enable Slack interactivity at /eve/v1/slack.
+# Subscribe the GitHub connector to pull_request events.
+# EVI_PR_REVIEW_SLACK_CHANNEL_ID=C...
+# EVI_SLACK_TEAM_ID=T...

apps/evi/agent/agent.ts

@@ -1,6 +1,7 @@
 import type { SessionAuthContext } from 'eve/context'
 import { defineAgent, defineDynamic } from 'eve'
 import { gatewayRouting, sessionTags } from './lib/gateway'
+import { reviewStepModel } from './lib/github/external-review'
 import { MODEL } from './lib/model'
 import { isScheduleAppAuth } from './lib/trust'
 
@@ -36,6 +37,14 @@ export default defineAgent({
     events: {
       'session.started': selectModel,
       'turn.started': selectModel,
+      'step.started': (_event, ctx) => {
+        const model = reviewStepModel(ctx.session.auth.current)
+        return {
+          model,
+          modelContextWindowTokens: typeof model === 'string' ? undefined : 1_000_000,
+          modelOptions: modelOptions(ctx),
+        }
+      },
     },
   }),
   reasoning: 'high',

apps/evi/agent/channels/github.ts

@@ -1,18 +1,20 @@
-import { defaultGitHubAuth, githubChannel } from 'eve/channels/github'
+import { defaultGitHubAuth } from 'eve/channels/github'
 import type { GitHubChannelState } from 'eve/channels/github'
+import { reviewGitHubChannel } from '../lib/github/review-trigger'
 import { githubCredentials } from '../lib/github/credentials'
 import { escalateFailedTriageQuietly, isAutonomousTriageState } from '../lib/github/escalate'
 import { failureComment } from '../lib/failure'
 import { isHomeRepository, repositoryOf } from '../lib/repo'
 import { AUTONOMOUS_GITHUB_PRINCIPAL, isAutonomous, MAINTAINER_GITHUB_ID, MAINTAINER_GITHUB_LOGIN } from '../lib/trust'
+import slack from './slack'
 
 const botName = 'evlogai'
 const mentionPattern = new RegExp(
   `@${botName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?=$|[^A-Za-z0-9_-])`,
   'iu',
 )
 
-export default githubChannel({
+export default reviewGitHubChannel({
   botName,
   credentials: githubCredentials,
   onComment: (ctx, comment) => {
@@ -59,7 +61,7 @@ export default githubChannel({
       )
     },
   },
-})
+}, slack)
 
 async function escalate(state: GitHubChannelState): Promise<void> {
   if (state.issueNumber === null) return

apps/evi/agent/channels/slack.ts

@@ -12,6 +12,7 @@ import { isMaintainer } from '../lib/trust'
 async function admitMaintainer(ctx: SlackInboundMessageContext, message: SlackMessage): Promise<SlackMentionResult> {
   const auth = defaultSlackAuth(message, ctx)
   if (auth === null || !isMaintainer(auth)) return null
+  if (ctx.slack.channelId === process.env.EVI_PR_REVIEW_SLACK_CHANNEL_ID) return null
   await ctx.thread.startTyping('Thinking…')
   return { auth }
 }

apps/evi/agent/extensions/github.ts

@@ -4,7 +4,7 @@ import { GITHUB_CONNECTOR } from '../lib/github/credentials'
 import { createLabelPolicy, writePolicy } from '../lib/github/label-approval'
 import { createPullRequestPolicy } from '../lib/github/pull-request-approval'
 import { homeRepository } from '../lib/repo'
-import { isAutonomous, MAINTAINER_GITHUB_LOGIN } from '../lib/trust'
+import { isAutonomous, isPullRequestReview, MAINTAINER_GITHUB_LOGIN } from '../lib/trust'
 
 /**
  * Every tool here is carried in the prompt on every turn, whatever the turn is
@@ -97,11 +97,12 @@ export default githubExtension({
   connector: GITHUB_CONNECTOR,
   context: homeRepository(),
   include: [...TOOLS],
-  // Omitted write tools keep the default always(): closeIssue, createPullRequestReview.
   // Connect scopes are derived from `include` (createLabel → issues:write) in sdk ≥ 1.11.1.
   requireApproval: {
     // Reversible and harmless on every kind of run; a card here only slows the PR flow down.
-    requestReviewers: (): ApprovalStatus => 'not-applicable',
+    requestReviewers: (ctx: ApprovalContext): ApprovalStatus => isPullRequestReview(ctx.session.auth.current) ? policy(ctx) : 'not-applicable',
+    closeIssue: (ctx: ApprovalContext): ApprovalStatus => isPullRequestReview(ctx.session.auth.current) ? policy(ctx) : 'user-approval',
+    createPullRequestReview: (ctx: ApprovalContext): ApprovalStatus => isPullRequestReview(ctx.session.auth.current) ? policy(ctx) : 'user-approval',
     createPullRequest: (ctx: ApprovalContext) => createPullRequestPolicy(ctx.session.auth.current, ctx.toolInput),
     updatePullRequest: policy,
     createIssue: autonomousWrite,

apps/evi/agent/lib/github/external-review.test.ts

@@ -0,0 +1,112 @@
+import { beforeEach, expect, it, vi } from 'vitest'
+import type { SessionAuthContext } from 'eve/context'
+import type { ApprovalContext, ApprovalResponseContext } from 'eve/tools/approval'
+import { canAccessAdminTools, canCaptureEvidence, isMaintainer, pullRequestReviewAuth, reviewState } from '../trust'
+import { writePolicy } from './label-approval'
+import { prepareExternalReview, publishExternalReview, reviewApprovalResponse, reviewExecutionPolicy, reviewScope } from './external-review'
+
+vi.mock('eve/context', () => ({
+  defineState: (_name: string, initial: () => unknown) => {
+    let value = initial()
+    return {
+      get: () => value,
+      update: (fn: (current: unknown) => unknown) => {
+        value = fn(value)
+      },
+    }
+  },
+}))
+
+const sha = 'a'.repeat(40)
+const auth = pullRequestReviewAuth({ owner: 'evloghq', repo: 'evlog', number: 12, sha })
+const pr = {
+  state: 'open', draft: false, title: 'Fix a bug',
+  user: { id: 999, login: 'contributor', type: 'User' },
+  author_association: 'NONE', head: { sha }, base: { sha: 'b'.repeat(40) },
+}
+const request = vi.fn()
+const run = vi.fn()
+const ctx = { getSandbox: vi.fn(() => Promise.resolve({ run })) }
+const github = { request }
+
+beforeEach(() => {
+  vi.stubEnv('VERCEL_ENV', 'production')
+  reviewState.update(() => ({ prepared: false, publishing: false, url: null }))
+  request.mockReset().mockResolvedValue(pr)
+  run.mockReset().mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' })
+  ctx.getSandbox.mockClear()
+})
+
+it('keeps review sessions below maintainer and admin authority', () => {
+  expect(isMaintainer(auth)).toBe(false)
+  expect(canAccessAdminTools(auth)).toBe(false)
+  expect(writePolicy(auth)).toMatchObject({ type: 'denied' })
+  expect(canCaptureEvidence(auth)).toBe(false)
+})
+
+it('refuses execution and publication before approved preparation', async () => {
+  expect(reviewExecutionPolicy({ session: { auth: { current: auth } } } as ApprovalContext)).toMatchObject({ type: 'denied' })
+  await expect(publishExternalReview(auth, { body: 'Review', comments: [] }, github)).rejects.toThrow('not prepared')
+  expect(request).not.toHaveBeenCalled()
+  expect(ctx.getSandbox).not.toHaveBeenCalled()
+})
+
+it.each(['approve', 'cancel'])('rejects an untrusted %s without changing review state', (decision) => {
+  const principal: SessionAuthContext = { authenticator: 'slack-webhook', principalType: 'user', principalId: 'slack:OTHER:U123', attributes: {} }
+  expect(reviewApprovalResponse({ response: { principal, decision } } as ApprovalResponseContext)).toMatchObject({ status: 'rejected' })
+  expect(reviewState.get().prepared).toBe(false)
+})
+
+it('requires a service identity and a home-repository scope', () => {
+  expect(() => reviewScope(null)).toThrow('PR review session')
+  expect(() => reviewScope({ ...auth, principalType: 'user' })).toThrow('PR review session')
+  expect(() => reviewScope(pullRequestReviewAuth({ owner: 'other', repo: 'repo', number: 12, sha }))).toThrow('not allowed')
+})
+
+it('fetches the exact approved head and unlocks execution after checkout', async () => {
+  await prepareExternalReview(auth, ctx, github)
+  expect(run).toHaveBeenCalledOnce()
+  expect(run.mock.calls[0]?.[0].command).toContain('refs/pull/12/head')
+  expect(run.mock.calls[0]?.[0].command).toContain(`git checkout --detach '${sha}'`)
+  expect(reviewExecutionPolicy({ session: { auth: { current: auth } } } as ApprovalContext)).toBe('not-applicable')
+  expect(canCaptureEvidence(auth)).toBe(true)
+})
+
+it.each([{ head: { sha: 'c'.repeat(40) } }, { state: 'closed' }, { draft: true }])('refuses checkout after PR changes: %j', async (change) => {
+  request.mockResolvedValue({ ...pr, ...change })
+  await expect(prepareExternalReview(auth, ctx, github)).rejects.toThrow('changed or closed')
+  expect(ctx.getSandbox).not.toHaveBeenCalled()
+  expect(reviewState.get().prepared).toBe(false)
+})
+
+it('keeps execution blocked after a failed checkout', async () => {
+  run.mockResolvedValue({ exitCode: 1, stdout: '', stderr: 'failed' })
+  await expect(prepareExternalReview(auth, ctx, github)).rejects.toThrow('checkout failed')
+  expect(reviewState.get().prepared).toBe(false)
+})
+
+it('publishes only a COMMENT review at the scoped commit and returns its receipt on repetition', async () => {
+  reviewState.update(state => ({ ...state, prepared: true }))
+  request.mockResolvedValueOnce(pr).mockResolvedValueOnce({ html_url: 'https://github.com/review' })
+  const input = { body: 'Verified regression', comments: [{ path: 'src/index.ts', line: 1, side: 'RIGHT' as const, body: '```suggestion\nfixed()\n```' }] }
+  expect(await publishExternalReview(auth, input, github)).toEqual({ url: 'https://github.com/review', alreadyPublished: false })
+  expect(request.mock.calls[1]).toEqual([expect.objectContaining({ number: 12, sha }), 'pulls/12/reviews', expect.objectContaining({ event: 'COMMENT', commit_id: sha, comments: input.comments })])
+  expect(request.mock.calls[1]?.[2].body).toBe(`Verified regression\n\nReviewed commit: \`${sha}\`. Suggestions require a maintainer to apply them.`)
+  expect(await publishExternalReview(auth, input, github)).toEqual({ url: 'https://github.com/review', alreadyPublished: true })
+  expect(request).toHaveBeenCalledTimes(2)
+})
+
+it('refuses publication at a changed head without making a GitHub write', async () => {
+  reviewState.update(state => ({ ...state, prepared: true }))
+  request.mockResolvedValue({ ...pr, head: { sha: 'c'.repeat(40) } })
+  await expect(publishExternalReview(auth, { body: 'Review', comments: [] }, github)).rejects.toThrow('changed or closed')
+  expect(request).toHaveBeenCalledOnce()
+})
+
+it('refuses another write after an ambiguous publication failure', async () => {
+  reviewState.update(state => ({ ...state, prepared: true }))
+  request.mockResolvedValueOnce(pr).mockRejectedValueOnce(new Error('network'))
+  await expect(publishExternalReview(auth, { body: 'Review', comments: [] }, github)).rejects.toThrow('network')
+  await expect(publishExternalReview(auth, { body: 'Review', comments: [] }, github)).rejects.toThrow('already attempted')
+  expect(request).toHaveBeenCalledTimes(2)
+})

apps/evi/agent/lib/github/external-review.ts

@@ -0,0 +1,154 @@
+import type { SessionAuthContext } from 'eve/context'
+import type { ToolContext } from 'eve/tools'
+import type { ApprovalContext, ApprovalResponseContext, ApprovalStatus } from 'eve/tools/approval'
+import { z } from 'zod'
+import { gateway, wrapLanguageModel } from 'ai'
+import { MODEL } from '../model'
+import { cloneUrl, homeRepository, isHomeRepository } from '../repo'
+import { isMaintainer, isPullRequestReview, reviewState } from '../trust'
+import { REPO_DIR, runOutput } from '../workspace'
+import { repositoryToken } from './credentials'
+
+export const pullRequestSchema = z.object({
+  state: z.string(),
+  draft: z.boolean(),
+  title: z.string(),
+  user: z.object({ id: z.number(), login: z.string(), type: z.string() }),
+  author_association: z.string(),
+  head: z.object({ sha: z.string().regex(/^[a-f0-9]{40}$/) }),
+  base: z.object({ sha: z.string().regex(/^[a-f0-9]{40}$/) }),
+})
+
+/** Limit preapproval model input and reject every call except native preparation. */
+export function reviewStepModel(auth: SessionAuthContext | null) {
+  if (!isPullRequestReview(auth) || reviewState.get().prepared) return MODEL
+  return wrapLanguageModel({
+    model: gateway(MODEL),
+    middleware: {
+      specificationVersion: 'v4',
+      transformParams({ params }) {
+        const attempted = params.prompt.some(message => message.role === 'tool'
+          && message.content.some(part => part.type === 'tool-result' && part.toolName === 'pr_review__prepare'))
+        return Promise.resolve({
+          ...params,
+          prompt: [{ role: 'user', content: [{ type: 'text', text: attempted ? 'The review did not start. Stop.' : 'Call pr_review__prepare to request approval. Do nothing else.' }] }],
+          tools: attempted ? [] : params.tools?.filter(tool => tool.name === 'pr_review__prepare'),
+          toolChoice: attempted ? { type: 'none' } : { type: 'tool', toolName: 'pr_review__prepare' },
+          maxOutputTokens: 256,
+        })
+      },
+      async wrapGenerate({ doGenerate, params }) {
+        const result = await doGenerate()
+        for (const part of result.content) {
+          if (part.type === 'tool-call' && (part.toolName !== 'pr_review__prepare' || params.toolChoice?.type === 'none')) throw new Error('Review work is blocked before approval.')
+        }
+        return result
+      },
+      async wrapStream({ doStream, params }) {
+        const result = await doStream()
+        return { ...result, stream: result.stream.pipeThrough(new TransformStream({
+          transform(part, controller) {
+            if ((part.type === 'tool-call' || part.type === 'tool-input-start') && (part.toolName !== 'pr_review__prepare' || params.toolChoice?.type === 'none')) throw new Error('Review work is blocked before approval.')
+            controller.enqueue(part)
+          },
+        })) }
+      },
+    },
+  })
+}
+
+/** Keep GitHub write credentials outside the contributor sandbox. */
+async function reviewGitHubRequest(record: { owner: string, repo: string }, suffix: string, body?: unknown): Promise<unknown> {
+  const token = await repositoryToken(record)
+  if (token === null) throw new Error('The GitHub App is not installed on the review repository.')
+  const response = await fetch(`https://api.github.com/repos/${record.owner}/${record.repo}/${suffix}`, {
+    method: body === undefined ? 'GET' : 'POST',
+    headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${token}`, 'X-GitHub-Api-Version': '2022-11-28', 'Content-Type': 'application/json' },
+    ...(body === undefined ? {} : { body: JSON.stringify(body) }),
+  })
+  if (!response.ok) throw new Error(`PR review GitHub request failed with status ${response.status}.`)
+  return response.json()
+}
+
+const scopeSchema = z.object({
+  owner: z.string(), repo: z.string(),
+  number: z.coerce.number().int().positive(),
+  sha: z.string().regex(/^[a-f0-9]{40}$/),
+})
+export type ReviewScope = z.infer<typeof scopeSchema>
+
+export const reviewInputSchema = z.object({
+  body: z.string().min(1).max(50_000),
+  comments: z.array(z.object({
+    path: z.string().min(1),
+    line: z.number().int().positive(),
+    side: z.enum(['LEFT', 'RIGHT']),
+    start_line: z.number().int().positive().optional(),
+    start_side: z.enum(['LEFT', 'RIGHT']).optional(),
+    body: z.string().min(1).max(10_000),
+  })).max(50),
+})
+export type ReviewInput = z.infer<typeof reviewInputSchema>
+
+/** Resolve the scope minted by the verified GitHub webhook, never model input. */
+export function reviewScope(auth: SessionAuthContext | null): ReviewScope {
+  if (!isPullRequestReview(auth) || auth === null) throw new Error('This tool requires a PR review session.')
+  const scope = scopeSchema.parse(auth.attributes)
+  if (!isHomeRepository({ owner: scope.owner, name: scope.repo })) throw new Error('PR review repository is not allowed.')
+  return scope
+}
+
+/** Native approval owns the pending request and authenticates both decisions. */
+export function reviewApprovalResponse({ response }: ApprovalResponseContext) {
+  return isMaintainer(response.principal)
+    ? { status: 'allowed' as const }
+    : { status: 'rejected' as const, reason: 'Only a trusted maintainer may decide this review.' }
+}
+
+/** Contributor code cannot run until the approved checkout has completed. */
+export function reviewExecutionPolicy({ session }: ApprovalContext): ApprovalStatus {
+  if (!isPullRequestReview(session.auth.current)) return 'not-applicable'
+  return reviewState.get().prepared ? 'not-applicable' : { type: 'denied', reason: 'Approve and prepare the PR before running contributor code.' }
+}
+
+export interface ReviewApi {
+  request(record: ReviewScope, suffix: string, body?: unknown): Promise<unknown>
+}
+const api: ReviewApi = { request: reviewGitHubRequest }
+
+async function currentPullRequest(scope: ReviewScope, github: ReviewApi) {
+  const pr = pullRequestSchema.parse(await github.request(scope, `pulls/${scope.number}`))
+  if (pr.state !== 'open' || pr.draft || pr.head.sha !== scope.sha) throw new Error('The PR changed or closed. A new head commit needs a new approval.')
+  return pr
+}
+
+/** Run only behind native approval and fetch exactly its server-held head. */
+export async function prepareExternalReview(auth: SessionAuthContext | null, ctx: { getSandbox(): Promise<Pick<Awaited<ReturnType<ToolContext['getSandbox']>>, 'run'>> }, github: ReviewApi = api) {
+  const scope = reviewScope(auth)
+  const pr = await currentPullRequest(scope, github)
+  const sandbox = await ctx.getSandbox()
+  const checkout = await sandbox.run({
+    command: `cd ${REPO_DIR} && git fetch ${cloneUrl(homeRepository())} refs/pull/${scope.number}/head && test "$(git rev-parse FETCH_HEAD)" = '${scope.sha}' && git checkout --detach '${scope.sha}'`,
+  })
+  if (checkout.exitCode !== 0) throw new Error(`Approved PR checkout failed: ${runOutput(checkout)}`)
+  reviewState.update(state => ({ ...state, prepared: true }))
+  return { repository: `${scope.owner}/${scope.repo}`, number: scope.number, revision: scope.sha, baseRevision: pr.base.sha, path: REPO_DIR }
+}
+
+/** Publish a COMMENT review at the approved head, with no branch mutation. */
+export async function publishExternalReview(auth: SessionAuthContext | null, input: ReviewInput, github: ReviewApi = api) {
+  const scope = reviewScope(auth)
+  const state = reviewState.get()
+  if (state.url !== null) return { url: state.url, alreadyPublished: true }
+  if (!state.prepared || state.publishing) throw new Error('Review is not prepared or publication was already attempted. Check GitHub before retrying.')
+  await currentPullRequest(scope, github)
+  if (reviewState.get().publishing) throw new Error('Review publication was already attempted. Check GitHub before retrying.')
+  reviewState.update(current => ({ ...current, publishing: true }))
+  const result = z.object({ html_url: z.string() }).parse(await github.request(scope, `pulls/${scope.number}/reviews`, {
+    commit_id: scope.sha, event: 'COMMENT',
+    body: `${input.body}\n\nReviewed commit: \`${scope.sha}\`. Suggestions require a maintainer to apply them.`,
+    comments: input.comments,
+  }))
+  reviewState.update(current => ({ ...current, url: result.html_url }))
+  return { url: result.html_url, alreadyPublished: false }
+}

apps/evi/agent/lib/github/label-approval.ts

@@ -1,6 +1,6 @@
 import type { SessionAuthContext } from 'eve/context'
 import type { ApprovalStatus } from 'eve/tools/approval'
-import { isAutonomous, isMaintainer } from '../trust'
+import { isAutonomous, isMaintainer, isPullRequestReview } from '../trust'
 
 const LABEL_NAME_MAX = 50
 const LABEL_DESCRIPTION_MAX = 100
@@ -15,6 +15,7 @@ const MULTILINE = /[\n\r\u2028\u2029]/
  * everyone else needs approval.
  */
 export function writePolicy(auth: SessionAuthContext | null): ApprovalStatus {
+  if (isPullRequestReview(auth)) return { type: 'denied', reason: 'PR review sessions may only publish their scoped review and suggestions.' }
   if (isAutonomous(auth)) {
     return {
       type: 'denied',