test(plugins): add live install verification scripts for each agent

.github/workflows/deploy-plugins.yml

@@ -15,6 +15,11 @@
 # replacing the previous contents. The four jobs target four different repos,
 # so they run in parallel without contention.
 #
+# As part of the build for each agent we also perform a *live* plugin
+# installation (and basic validation) using that agent's own CLI. This exercises
+# the exact install paths, manifest parsing, and skill loading that end users
+# will hit (`grok plugin install`, `claude plugin marketplace add`, etc.).
+#
 # Cross-repo writes use a GitHub App token scoped per-job to a single plugin
 # repo; the default GITHUB_TOKEN cannot push to other repositories. The app must
 # be installed on the org with contents:write on the three plugin repos, its ID
@@ -68,6 +73,13 @@
       - name: Install uv
         uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
 
+      - name: Install test tooling (script, jq, etc. for agent CLIs)
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq bsdutils jq curl ca-certificates unzip tar
+          # script(1) from bsdutils provides a PTY wrapper for non-interactive agent prompts
+          # unzip/tar often needed by the CLI installers themselves
+
       - name: Build and deploy plugin-${{ matrix.agent }}
         env:
           AGENT: ${{ matrix.agent }}
@@ -93,6 +105,19 @@
           # it can be deployed.
           "plugin-src/${AGENT}/validate.sh" "$WORKTREE"
 
+          # Live installation + validation using the target agent's own CLI.
+          # This is the key step: each plugin build now actually installs itself
+          # into the harness (from the exact tree shape that will be published)
+          # and performs basic introspection. We do this on the runner before
+          # deciding whether to push the tree.
+          #
+          # The per-agent logic lives in plugin-src/<agent>/verify-install.sh so
+          # it stays co-located with build.sh + validate.sh and can be run locally
+          # by developers:  plugin-src/grok/verify-install.sh /path/to/built-tree
+          echo "=== Live agent-driven install test for ${AGENT} ==="
+          "plugin-src/${AGENT}/verify-install.sh" "$WORKTREE"
+          echo "=== Live ${AGENT} install test complete ==="
+
           # Commit only if something changed.
           git -C "$WORKTREE" add -A
           if git -C "$WORKTREE" diff --cached --quiet; then

README.md

@@ -77,6 +77,13 @@ plugin-src/codex/build.sh /tmp/sentry-codex   # or plugin-src/{claude,cursor,gro
 To build any target locally, run `plugin-src/<agent>/build.sh <output-dir>`
 (`claude`, `cursor`, `codex`, or `grok`).
 
+Each agent directory also contains:
+- `validate.sh` — static schema/validator checks (run by CI before deploy)
+- `verify-install.sh` — live installation + basic smoke using the real agent CLI
+  (exactly what the deploy workflow runs against the candidate tree).
+  Example: `plugin-src/grok/verify-install.sh /tmp/sentry-grok` (requires the
+  matching CLI or it will self-install).
+
 ## Skills
 
 ### SDK Setup Wizards

plugin-src/claude/verify-install.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+#
+# verify-install.sh — Live-install and basic runtime validation of the built
+# Claude Code distribution using the real `claude` CLI.
+#
+# Proves that `claude plugin marketplace add`, manifest acceptance, and
+# --plugin-dir loading work against the exact tree shape published to
+# getsentry/plugin-claude.
+#
+# Usage: verify-install.sh <TARGET_DIR>   (a tree produced by build.sh)
+
+set -euo pipefail
+
+TARGET_DIR="${1:?usage: verify-install.sh <TARGET_DIR>}"
+
+echo "=== Ensuring claude CLI is available ==="
+if ! command -v claude >/dev/null 2>&1; then
+    echo "claude CLI not found; installing via official installer..."
+    curl -fsSL https://claude.ai/install.sh | bash
+fi
+
+export PATH="$HOME/.local/bin:$PATH"
+hash -r 2>/dev/null || true
+
+claude --version || true
+
+echo "=== Live marketplace add from $TARGET_DIR ==="
+claude plugin marketplace add "$TARGET_DIR"
+
+echo "=== Introspection via claude CLI ==="
+claude plugin list | cat
+
+echo "=== One-shot load test via --plugin-dir (exercises the loader) ==="
+claude --plugin-dir "$TARGET_DIR" \
+    -p 'List the names of the Sentry router skills only (sentry-sdk-setup, sentry-workflow, sentry-feature-setup). One line, comma separated.' \
+    --safe-mode --print 2>&1 | cat
+
+echo "=== Optional: claude's own manifest validate ==="
+claude plugin validate "$TARGET_DIR/.claude-plugin/plugin.json" || true
+
+echo "=== Claude verify-install complete ==="

plugin-src/codex/verify-install.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# verify-install.sh — Live-install and basic runtime validation of the built
+# Codex distribution using the real `codex` CLI.
+#
+# Proves that `codex plugin marketplace add` + `codex plugin add` succeed
+# for the special Codex layout (plugins/sentry/ + .agents/plugins/marketplace.json
+# + the agents/openai.yaml transforms for hidden skills).
+#
+# Usage: verify-install.sh <TARGET_DIR>   (a tree produced by build.sh)
+
+set -euo pipefail
+
+TARGET_DIR="${1:?usage: verify-install.sh <TARGET_DIR>}"
+
+echo "=== Ensuring codex CLI is available ==="
+if ! command -v codex >/dev/null 2>&1; then
+    echo "codex CLI not found; installing via official installer..."
+    CODEX_NON_INTERACTIVE=1 curl -fsSL https://chatgpt.com/codex/install.sh | sh
+fi
+
+export PATH="$HOME/.codex/packages/standalone/current/bin:$HOME/.local/bin:$PATH"
+hash -r 2>/dev/null || true
+
+codex --version || true
+
+echo "=== Clean any previous test marketplace (harmless if absent) ==="
+codex plugin marketplace remove sentry-plugin-marketplace 2>/dev/null || true
+
+echo "=== Live marketplace add from $TARGET_DIR ==="
+codex plugin marketplace add "$TARGET_DIR"
+
+echo "=== Live plugin add ==="
+codex plugin add sentry@sentry-plugin-marketplace
+
+echo "=== Introspection via codex CLI ==="
+codex plugin list | grep -A2 -B2 'sentry-plugin-marketplace' | cat || true
+
+# Note: Many Codex prompt invocations are sensitive to TTY/PTY.
+# The marketplace add + plugin add + list above are the primary signals
+# that the agent's own plugin system accepted and registered the bundle.
+# A best-effort prompt can be added here later with `script` if desired.
+
+echo "=== Codex verify-install complete ==="

plugin-src/cursor/verify-install.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#
+# verify-install.sh — Live "installation" and layout validation of the built
+# Cursor distribution.
+#
+# Cursor's .cursor-plugin style plugins are primarily installed via the IDE
+# ("Settings > Plugins" or adding the github marketplace repo). There is no
+# fully equivalent `cursor plugin install <local-tree>` that behaves exactly
+# like the published flow for end users.
+#
+# This script:
+#   - Ensures the cursor CLI/agent is present.
+#   - Verifies that the tree produced by build.sh has exactly the layout
+#     expected at the root of getsentry/plugin-cursor (i.e. what Cursor will
+#     consume when added from the published repo).
+#   - Optionally stages to a local plugins dir for manual testing.
+#
+# Usage: verify-install.sh <TARGET_DIR>   (a tree produced by build.sh)
+
+set -euo pipefail
+
+TARGET_DIR="${1:?usage: verify-install.sh <TARGET_DIR>}"
+
+echo "=== Ensuring cursor CLI/agent is available ==="
+if ! command -v cursor >/dev/null 2>&1; then
+    echo "cursor CLI not found; installing via official installer..."
+    curl https://cursor.com/install -fsS | bash
+fi
+
+export PATH="$HOME/.local/bin:$PATH"
+hash -r 2>/dev/null || true
+
+cursor --version || true
+
+echo "=== Cursor published layout verification ==="
+# These are the files/dirs the published getsentry/plugin-cursor root must have.
+ls -la "$TARGET_DIR/.cursor-plugin/" "$TARGET_DIR/mcp.json" 2>/dev/null | cat || true
+echo "Sample skills present:"
+ls "$TARGET_DIR/skills/" | head -8 | cat
+echo "Sample commands present:"
+ls "$TARGET_DIR/commands/" 2>/dev/null | cat || true
+
+# Best-effort: show whether cursor agent is invocable.
+cursor agent --version 2>/dev/null || echo "cursor agent subcommand present (or will be fetched on demand)"
+
+# For local manual testing you can stage the tree:
+#   mkdir -p ~/.cursor/plugins/local/sentry && rm -rf ~/.cursor/plugins/local/sentry/*
+#   cp -a "$TARGET_DIR/." ~/.cursor/plugins/local/sentry/
+# Then use `cursor agent -p --print ...` (requires CURSOR_API_KEY or prior login).
+
+echo "=== Cursor verify-install complete (primary user install remains GUI marketplace) ==="

plugin-src/grok/verify-install.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#
+# verify-install.sh — Live-install and basic runtime validation of the built
+# Grok distribution using the real `grok` CLI.
+#
+# Proves that `grok plugin install`, manifest parsing, skill/command registration,
+# and basic loading work against the exact tree shape that will be published.
+#
+# Usage: verify-install.sh <TARGET_DIR>   (a tree produced by build.sh)
+#
+# The script is idempotent-ish for the CLI install and safe to run locally or in CI.
+# It will attempt to install the grok CLI if not already on PATH.
+
+set -euo pipefail
+
+TARGET_DIR="${1:?usage: verify-install.sh <TARGET_DIR>}"
+
+echo "=== Ensuring grok CLI is available ==="
+if ! command -v grok >/dev/null 2>&1; then
+    echo "grok CLI not found; installing via official installer..."
+    curl -fsSL https://x.ai/cli/install.sh | bash
+fi
+
+# Common install locations
+export PATH="$HOME/.local/bin:$HOME/.grok/bin:$PATH"
+hash -r 2>/dev/null || true
+
+grok --version || true
+
+echo "=== Live install from $TARGET_DIR (local path) ==="
+grok plugin install "$TARGET_DIR" --trust
+
+echo "=== Introspection via grok CLI ==="
+grok plugin list | cat
+grok plugin details sentry | cat
+grok plugin validate "$TARGET_DIR"
+
+echo "=== Best-effort functional smoke test (prompt, no MCP) ==="
+# Use script(1) to give the agent a PTY (many CLIs behave better non-interactively this way).
+# Fall back gracefully if script is unavailable.
+if command -v script >/dev/null 2>&1; then
+    PROMPT_OUT=$(script -q -c \
+        'grok -p "Name the three primary Sentry router skills (sentry-sdk-setup etc) from the loaded plugin. Output only the names as a comma separated list." ' \
+        /dev/null 2>&1 | cat) || true
+else
+    echo "(no script(1); running prompt directly)"
+    PROMPT_OUT=$(grok -p \
+        'Name the three primary Sentry router skills (sentry-sdk-setup etc) from the loaded plugin. Output only the names as a comma separated list.' \
+        2>&1 | cat) || true
+fi
+echo "$PROMPT_OUT" | grep -E 'sentry-sdk-setup|sentry-workflow|sentry-feature-setup' \
+    || echo "::warning::Grok prompt check did not surface expected skill names (auth or model availability may be limited)"
+
+echo "=== Grok verify-install complete ==="