Add --plugin-dir and show what the cache saved

README.md

@@ -45,6 +45,9 @@ skillcheck run --only 2,5     # confirm what the batch flagged with real runs
 Every option, the default file lookup and exit codes are in
 [Commands](docs/commands.md).
 
+To test a plugin from its repository, use `--config-dir` with an empty
+directory and `--plugin-dir .`; cases name its skills as `plugin:skill`.
+
 No cases yet? `skillcheck gen -o skillcheck.yaml` lets the model draft them from
 your skill descriptions, for you to review. Coming from Anthropic's
 skill-creator? `skillcheck import eval_set.json --skill <name>` converts its

action.yml

@@ -15,6 +15,9 @@ inputs:
       Claude Code config dir, so the run sees only them. Empty uses the
       runner's own config.
     default: skills
+  plugin-dir:
+    description: Directory of a Claude Code plugin to test from source; empty auto-detects .claude-plugin/plugin.json at the repository root.
+    default: ""
   lint:
     description: Run the free lint step first.
     default: "true"
@@ -94,15 +97,27 @@ runs:
       run: npm install -g "@anthropic-ai/claude-code@$CC_VERSION" "@icntswm/skillcheck@$SC_VERSION"
 
     - name: config dir
-      if: inputs.skills-dir != ''
       shell: bash
       env:
         SKILLS_DIR: ${{ inputs.skills-dir }}
+        PLUGIN_DIR: ${{ inputs.plugin-dir }}
       run: |
         dir="$RUNNER_TEMP/skillcheck-config"
-        mkdir -p "$dir/skills"
-        cp -R "$SKILLS_DIR/." "$dir/skills/"
-        echo "SKILLCHECK_CONFIG_DIR=$dir" >> "$GITHUB_ENV"
+        if [ -n "$PLUGIN_DIR" ] || [ -f ".claude-plugin/plugin.json" ]; then
+          rm -rf "$dir"
+          mkdir -p "$dir"
+          echo "SKILLCHECK_CONFIG_DIR=$dir" >> "$GITHUB_ENV"
+          echo "SKILLCHECK_PLUGIN_DIR=${PLUGIN_DIR:-.}" >> "$GITHUB_ENV"
+        elif [ -n "$SKILLS_DIR" ]; then
+          mkdir -p "$dir/skills"
+          cp -R "$SKILLS_DIR/." "$dir/skills/"
+          echo "SKILLCHECK_CONFIG_DIR=$dir" >> "$GITHUB_ENV"
+          echo "SKILLCHECK_PLUGIN_DIR=" >> "$GITHUB_ENV"
+        else
+          # GITHUB_ENV lasts the whole job: clear what an earlier call of this action set
+          echo "SKILLCHECK_CONFIG_DIR=" >> "$GITHUB_ENV"
+          echo "SKILLCHECK_PLUGIN_DIR=" >> "$GITHUB_ENV"
+        fi
 
     - name: lint
       if: inputs.lint == 'true'
@@ -112,6 +127,9 @@ runs:
       run: |
         args=(lint "$FILE")
         [ -n "$SKILLCHECK_CONFIG_DIR" ] && args+=(--config-dir "$SKILLCHECK_CONFIG_DIR")
+        if [ -n "$SKILLCHECK_PLUGIN_DIR" ]; then
+          if skillcheck --help | grep -- --plugin-dir > /dev/null; then args+=(--plugin-dir "$SKILLCHECK_PLUGIN_DIR"); else echo "::warning::this skillcheck version has no --plugin-dir"; fi
+        fi
         skillcheck "${args[@]}"
 
     - name: run
@@ -131,6 +149,9 @@ runs:
       run: |
         args=(run "$FILE")
         [ -n "$SKILLCHECK_CONFIG_DIR" ] && args+=(--config-dir "$SKILLCHECK_CONFIG_DIR")
+        if [ -n "$SKILLCHECK_PLUGIN_DIR" ]; then
+          if skillcheck --help | grep -- --plugin-dir > /dev/null; then args+=(--plugin-dir "$SKILLCHECK_PLUGIN_DIR"); else echo "::warning::this skillcheck version has no --plugin-dir"; fi
+        fi
         [ "$BATCH" = "true" ] && args+=(--batch)
         [ -n "$MODEL" ] && args+=(--model "$MODEL")
         [ -n "$REPEAT" ] && args+=(--repeat "$REPEAT")

docs/ci.md

@@ -151,7 +151,7 @@ does not block unrelated pull requests, but it still shows in the comment.
 when nothing that affects its routing changed: the case itself, the frontmatter
 of any skill or command, `CLAUDE.md` files, the model, the timeout, the agent
 and its version. It does not work with `batch: true`. Failed and skipped cases
-always run again. Reused cases are marked cached and cost nothing.
+always run again. Reused cases are marked cached and cost nothing; the report shows what the cache saved.
 
 Editing a skill's body (everything after the frontmatter) does not invalidate
 the cache, since only descriptions and other frontmatter affect routing.
@@ -189,6 +189,17 @@ A failing case fails the step, so a step that reads the outputs needs
 `if: always()`. `skillcheck.xml` still works with `actions/upload-artifact`
 or a JUnit reporter action.
 
+### Testing a plugin
+
+Set the action's `plugin-dir` input to a plugin source directory. When it is
+empty, the action auto-detects `.claude-plugin/plugin.json` at the repository
+root. Plugin mode uses an empty isolated config directory, and cases refer to
+skills as `plugin:skill`. The `skills-dir` input is ignored in plugin mode.
+
+```yaml
+- uses: icntswm/skillcheck@v1
+```
+
 ### Only the skills a pull request touched
 
 `--skill` keeps the cases that mention the given skills, so a pull request

docs/commands.md

@@ -153,6 +153,7 @@ purpose:
 | `--no-early-stop` | | let the agent finish its turn instead of stopping it once a skill is picked; costs more; not with `--batch` |
 | `--budget <usd>` | | stop starting new runs once the spend estimate reaches this; the rest is reported as skipped |
 | `--config-dir <dir>` | | see [Where skills come from](#where-skills-come-from) |
+| `--plugin-dir <dir>` | | load a plugin from source for this session; repeatable |
 
 **Reports**
 

src/agents/claude.ts

@@ -213,6 +213,7 @@ function claudeArgs(opts: RunOptions): string[] {
     "--disallowedTools", ...DISALLOWED_TOOLS,
   ];
   if (opts.model) args.push("--model", opts.model);
+  for (const dir of opts.pluginDirs ?? []) args.push("--plugin-dir", dir);
   return args;
 }
 
@@ -230,20 +231,23 @@ function batchArgs(opts: BatchOptions): string[] {
     "--disallowedTools", ...DISALLOWED_TOOLS,
   ];
   if (opts.model) args.push("--model", opts.model);
+  for (const dir of opts.pluginDirs ?? []) args.push("--plugin-dir", dir);
   return args;
 }
 
-function listArgs(): string[] {
+function listArgs(opts: { pluginDirs?: string[] }): string[] {
   // The init event arrives before any model call, so the process is killed
   // before anything is billed; the disallow list is a guard for that race.
-  return [
+  const args = [
     "-p", "none",
     "--output-format", "stream-json",
     "--verbose",
     "--max-turns", "1",
     "--permission-mode", "plan",
     "--disallowedTools", ...DISALLOWED_TOOLS,
   ];
+  for (const dir of opts.pluginDirs ?? []) args.push("--plugin-dir", dir);
+  return args;
 }
 
 /** Seconds as written in the timeout message: 180000 -> "180", 400 -> "0.4". */
@@ -385,11 +389,11 @@ export class ClaudeAdapter implements AgentAdapter {
   }
 
   /** Ask the agent for its skill lists; killed right after init, so no model call. */
-  async listSkills(opts: { configDir?: string; timeoutMs: number }): Promise<SkillList> {
+  async listSkills(opts: { configDir?: string; pluginDirs?: string[]; timeoutMs: number }): Promise<SkillList> {
     const bin = process.env.SKILLCHECK_CLAUDE_BIN || "claude";
     const workdir = await mkdtemp(path.join(os.tmpdir(), "skillcheck-"));
     try {
-      const out = await this.streamRun(bin, listArgs(), workdir, opts.timeoutMs, {
+      const out = await this.streamRun(bin, listArgs(opts), workdir, opts.timeoutMs, {
         earlyStop: false,
         configDir: opts.configDir,
         stop: (stream) => stream.sawInit,

src/agents/types.ts

@@ -6,6 +6,7 @@ export interface RunOptions {
   earlyStop: boolean;
   /** claude config dir, passed to the child as CLAUDE_CONFIG_DIR */
   configDir?: string;
+  pluginDirs?: string[];
 }
 
 /** Billed tokens of one run, as the API reports them. */
@@ -42,6 +43,7 @@ export interface BatchOptions {
   timeoutMs: number;
   /** claude config dir, passed to the child as CLAUDE_CONFIG_DIR */
   configDir?: string;
+  pluginDirs?: string[];
 }
 
 export interface BatchResult {
@@ -69,5 +71,5 @@ export interface AgentAdapter {
   /** one call answers many cases; agents without batch mode omit it */
   runBatch?(opts: BatchOptions): Promise<BatchResult>;
   /** ask the agent what it can load; agents without this omit it */
-  listSkills?(opts: { configDir?: string; timeoutMs: number }): Promise<SkillList>;
+  listSkills?(opts: { configDir?: string; pluginDirs?: string[]; timeoutMs: number }): Promise<SkillList>;
 }

src/cache.ts

@@ -88,6 +88,8 @@ export function routingContext(
     kind: doc.kind,
     name: doc.name,
     plugin: doc.plugin,
+    // inline manifest commands have no frontmatter of their own
+    description: doc.description,
     frontmatter: rawFrontmatter(readFile, doc.file),
   }));
   for (const root of roots) {

src/cases.ts

@@ -2,7 +2,7 @@ import { readFile } from "node:fs/promises";
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { parse as parseYaml } from "yaml";
-import { commandNames, skillDirNames, skillRoots } from "./describe.js";
+import { commandNames, loadSkillDocs, skillDirNames, skillRoots } from "./describe.js";
 
 export interface Case {
   index: number; // 1-based position in file
@@ -217,24 +217,25 @@ export function findDefaultCasesFile(cwd: string): string | undefined {
 }
 
 /** Names of skills/commands discoverable on disk, plus built-in slash skills. */
-export function knownSkillNames(opts?: { home?: string; cwd?: string; configDir?: string }): Set<string> {
+export function knownSkillNames(opts?: { home?: string; cwd?: string; configDir?: string; pluginDirs?: string[] }): Set<string> {
   const out = new Set<string>(BUILTIN_SKILLS);
   for (const root of skillRoots(opts)) {
     for (const name of skillDirNames(path.join(root, "skills"))) out.add(name);
     for (const name of commandNames(path.join(root, "commands"))) out.add(name);
   }
+  for (const doc of loadSkillDocs(opts)) if (doc.plugin !== null) out.add(doc.name);
   return out;
 }
 
 const NAME_FIELDS = ["expect", "expect_any", "forbid", "first"] as const;
 
-export function unknownNames(suite: Suite, known: Set<string>): string[] {
+export function unknownNames(suite: Suite, known: Set<string>, sourcePlugins: Set<string> = new Set()): string[] {
   const errors: string[] = [];
   for (const c of suite.cases) {
     for (const field of NAME_FIELDS) {
       const names = field === "first" ? (c.first ? [c.first] : []) : c[field];
       for (const name of names) {
-        if (name.includes(":")) continue; // plugin skills are not checked
+        if (name.includes(":") && !sourcePlugins.has(name.slice(0, name.indexOf(":")))) continue; // installed plugin skills are not checked
         if (!known.has(name)) errors.push(`#${c.index}: unknown skill "${name}" in ${field}`);
       }
     }

src/cli.ts

@@ -22,7 +22,7 @@ import {
   type Suite,
 } from "./cases.js";
 import { confusion } from "./confusion.js";
-import { loadSkillDocs, pluginInstallPaths, skillRoots, type SkillDoc } from "./describe.js";
+import { loadSkillDocs, pluginInstallPaths, skillRoots, sourcePlugins, type SkillDoc } from "./describe.js";
 import { GEN_SCHEMA, buildGenPrompt, caseEntry, genSuite, orderCases, parseGenAnswer, type GenCase } from "./gen.js";
 import { aggregate, judge, type CaseResult, type RunVerdict } from "./judge.js";
 import { evalSetToSuite, type ImportedSuite } from "./import.js";
@@ -80,6 +80,8 @@ list/init options:
       --timeout <sec>    per-call timeout (same default as run)
       --config-dir <dir> claude config dir: run/check/lint/list/init use it instead of
                          CLAUDE_CONFIG_DIR; run, list and init also set it for the agent
+      --plugin-dir <dir> plugin source directory (repeatable); run/check/lint/list/init/gen
+                         load plugins from it for this session
 init options:
       --force            overwrite an existing file
 import options:
@@ -134,14 +136,15 @@ const OPTIONS = {
   overlap: { type: "string" },
   strict: { type: "boolean", default: false },
   "config-dir": { type: "string" },
+  "plugin-dir": { type: "string", multiple: true },
   force: { type: "boolean", default: false },
   out: { type: "string", short: "o" },
   "per-skill": { type: "string" },
   plugin: { type: "string" },
   append: { type: "string" },
 } satisfies ParseArgsOptionsConfig;
 
-type Values = { [K in keyof typeof OPTIONS]: (typeof OPTIONS)[K]["type"] extends "boolean" ? boolean : string | undefined };
+type Values = { [K in keyof typeof OPTIONS]: (typeof OPTIONS)[K]["type"] extends "boolean" ? boolean : string | undefined } & { "plugin-dir": string[] | undefined };
 
 export interface Io {
   stdout: ReportStream;
@@ -181,6 +184,7 @@ interface Flags {
   overlap: number;
   strict: boolean;
   configDir?: string;
+  pluginDirs: string[];
   force: boolean;
   out?: string;
   perSkill: number;
@@ -228,12 +232,21 @@ function parseFlags(values: Values, cwd: string): Flags {
     overlap: ratioFlag(values.overlap) ?? 0.3,
     strict: values.strict,
     configDir: values["config-dir"] !== undefined ? resolveConfigDir(values["config-dir"], cwd) : undefined,
+    pluginDirs: (values["plugin-dir"] ?? []).map((dir) => resolvePluginDir(dir, cwd)),
     force: values.force,
     out: values.out,
     perSkill: intFlag(values["per-skill"], "per-skill", 1) ?? 4,
   };
 }
 
+function resolvePluginDir(raw: string, cwd: string): string {
+  const abs = path.resolve(cwd, raw);
+  let isDir = false;
+  try { isDir = fs.statSync(abs).isDirectory(); } catch { /* missing */ }
+  if (!isDir) throw new UsageError(`--plugin-dir is not a directory: ${abs}`);
+  return abs;
+}
+
 /** --config-dir: absolute, must be an existing directory. */
 function resolveConfigDir(raw: string, cwd: string): string {
   const abs = path.resolve(cwd, raw);
@@ -366,7 +379,7 @@ async function checkCommand(file: string, flags: Flags, io: Io): Promise<number>
   if (!suite) return 2;
   const cases = applySkillFilter(suite.cases, flags.skill);
   if (flags.nameCheck) {
-    const errors = unknownNames({ ...suite, cases }, knownSkillNames({ cwd: io.cwd, configDir: flags.configDir }));
+    const errors = unknownNames({ ...suite, cases }, knownSkillNames({ cwd: io.cwd, configDir: flags.configDir, pluginDirs: flags.pluginDirs }), new Set(sourcePlugins(flags.pluginDirs).map((p) => p.plugin)));
     if (errors.length > 0) {
       printConfigError(io.stderr, file, new ConfigError(errors));
       return 2;
@@ -385,7 +398,7 @@ async function lintCommand(positional: string | undefined, flags: Flags, io: Io)
   } else {
     suite = null;
   }
-  const docs = loadSkillDocs({ cwd: io.cwd, configDir: flags.configDir });
+  const docs = loadSkillDocs({ cwd: io.cwd, configDir: flags.configDir, pluginDirs: flags.pluginDirs });
   const report = lint(docs, suite, { top: flags.top, overlap: flags.overlap, minLength: LINT_DEFAULTS.minLength });
   printLint(io.stdout, report, docs, suite);
   if (flags.strict && (report.short.length || report.similar.length || report.far.length)) return 1;
@@ -399,7 +412,7 @@ async function listCommand(flags: Flags, io: Io, deps: { adapter?: AgentAdapter
     io.stderr.write(`skillcheck: agent ${adapter.name} has no list mode\n`);
     return 2;
   }
-  const list = await adapter.listSkills({ configDir: flags.configDir, timeoutMs: flags.timeoutSec * 1000 });
+  const list = await adapter.listSkills({ configDir: flags.configDir, pluginDirs: flags.pluginDirs, timeoutMs: flags.timeoutSec * 1000 });
   if (list.error) {
     io.stderr.write(`skillcheck: ${list.error}\n`);
     return 2;
@@ -464,7 +477,7 @@ async function initCommand(
   let skills: string[] = [];
   let slashCommands: string[] = [];
   const list: SkillList | null = adapter.listSkills
-    ? await adapter.listSkills({ configDir: flags.configDir, timeoutMs: flags.timeoutSec * 1000 })
+    ? await adapter.listSkills({ configDir: flags.configDir, pluginDirs: flags.pluginDirs, timeoutMs: flags.timeoutSec * 1000 })
     : null;
   if (list && !list.error) {
     skills = list.skills;
@@ -473,7 +486,7 @@ async function initCommand(
     // a missing or failing agent must not block writing the starter file
     const why = list?.error ?? `agent ${adapter.name} has no list mode`;
     io.stderr.write(`note: could not ask the agent (${why}); listed skills found on disk\n`);
-    skills = [...knownSkillNames({ cwd: io.cwd, configDir: flags.configDir })].sort();
+    skills = [...knownSkillNames({ cwd: io.cwd, configDir: flags.configDir, pluginDirs: flags.pluginDirs })].sort();
   }
   try {
     fs.writeFileSync(file, initTemplate(flags.agent ?? "claude", skills, slashCommands));
@@ -523,7 +536,7 @@ function importCommand(positional: string | undefined, flags: Flags, io: Io): nu
 }
 
 async function genCommand(flags: Flags, io: Io, deps: { adapter?: AgentAdapter }): Promise<number> {
-  const docs = loadSkillDocs({ cwd: io.cwd, configDir: flags.configDir });
+  const docs = loadSkillDocs({ cwd: io.cwd, configDir: flags.configDir, pluginDirs: flags.pluginDirs });
   const byName = new Map(docs.map((doc) => [doc.name, doc]));
   const append = flags.append === undefined ? null : path.resolve(io.cwd, flags.append);
   const existing = append === null ? null : await loadCases(append, io);
@@ -571,6 +584,7 @@ async function genCommand(flags: Flags, io: Io, deps: { adapter?: AgentAdapter }
         model: flags.model,
         timeoutMs: flags.timeoutSec * 1000,
         configDir: flags.configDir,
+        pluginDirs: flags.pluginDirs,
       });
       if (result.error) {
         io.stderr.write(`skillcheck: gen failed for ${group.map((doc) => doc.name).join(", ")}: ${result.error}\n`);
@@ -793,10 +807,10 @@ async function runCommand(file: string, flags: Flags, io: Io, deps: { adapter?:
   }
   const selected = applySkillFilter(byOnly, flags.skill);
 
-  const docs = flags.cacheFile === undefined ? [] : loadSkillDocs({ cwd: io.cwd, configDir: flags.configDir });
+  const docs = flags.cacheFile === undefined ? [] : loadSkillDocs({ cwd: io.cwd, configDir: flags.configDir, pluginDirs: flags.pluginDirs });
   let context = "";
   if (flags.cacheFile !== undefined) {
-    const where = { cwd: io.cwd, configDir: flags.configDir };
+    const where = { cwd: io.cwd, configDir: flags.configDir, pluginDirs: flags.pluginDirs };
     const [userRoot, projectRoot] = skillRoots(where) as [string, string];
     // nested commands/ dirs, which loadSkillDocs does not list
     const roots = [
@@ -890,6 +904,7 @@ async function runCommand(file: string, flags: Flags, io: Io, deps: { adapter?:
       fixed: report.cases.filter((c) => c.change === "fixed").map((c) => `#${c.id ?? c.index}`),
     } : undefined,
     cached: report.summary.cached,
+    savedUsd: report.summary.savedUsd,
   });
   if (flags.batch) {
     // cases that only errored have no answer to confirm
@@ -976,6 +991,7 @@ async function runIndividually(
       timeoutMs: flags.timeoutSec * 1000,
       earlyStop: flags.earlyStop,
       configDir: flags.configDir,
+      pluginDirs: flags.pluginDirs,
     });
     fatal.note(r.error);
     if (r.availableSkills) {
@@ -1026,7 +1042,7 @@ async function runBatched(
     const active = job.chunk.filter((item) => job.round < item.runs.length);
     if (active.length === 0) return;
     const prompt = buildBatchPrompt(active.map((item, i) => ({ n: i + 1, query: item.c.query })));
-    const br = await runBatch({ prompt, schema: BATCH_SCHEMA, model, timeoutMs: flags.timeoutSec * 1000, configDir: flags.configDir });
+    const br = await runBatch({ prompt, schema: BATCH_SCHEMA, model, timeoutMs: flags.timeoutSec * 1000, configDir: flags.configDir, pluginDirs: flags.pluginDirs });
     const parsed = parseBatchAnswer(br.structured, br.text);
     const chunkError = br.error ?? parsed.error; // a chunk error is every case's error
     fatal.note(br.error);