fix: report the failure stage and a real version in action telemetry

package.json

@@ -2,6 +2,7 @@
   "name": "@evlog/action",
   "private": true,
   "type": "module",
+  "version": "1.3.0",
   "description": "GitHub Action for evlog: score the observability of your entry points and put the findings on the pull request.",
   "repository": "github:evloghq/action",
   "license": "MIT",

src/lib/telemetry.mjs

@@ -1,4 +1,5 @@
 import { execFile } from 'node:child_process'
+import { readFileSync } from 'node:fs'
 import { mkdtemp, rm } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
@@ -14,6 +15,17 @@ export const COLLECT_FIELDS = {
   baselineMode: ['none', 'base', 'spec'],
   checkOutcome: ['disabled', 'created', 'skipped', 'failed'],
   commentOutcome: ['disabled', 'created', 'updated', 'skipped', 'failed'],
+  errorStage: ['inputs', 'baseline', 'cli', 'check', 'comment', 'unknown'],
+}
+
+/** The runner leaves GITHUB_ACTION_REF empty for composite actions; the action itself is checked out at the pinned ref. */
+export function resolveVersion(env) {
+  if (env.GITHUB_ACTION_REF) return env.GITHUB_ACTION_REF
+  try {
+    const { version } = JSON.parse(readFileSync(join(env.GITHUB_ACTION_PATH ?? '', 'package.json')))
+    if (version) return version
+  } catch {}
+  return 'local'
 }
 
 /** Install the pinned SDK outside the workspace, without package scripts. */
@@ -62,7 +74,7 @@ export async function runWithTelemetry(inputs, work, {
     loaded = await load(env)
     handle = loaded.sdk.createGitHubActionsTelemetry({
       name: 'evlog-action',
-      version: env.GITHUB_ACTION_REF || 'local',
+      version: resolveVersion(env),
       environment: 'production',
       actionName: 'evloghq/action',
       endpoint: ENDPOINT,
@@ -80,10 +92,12 @@ export async function runWithTelemetry(inputs, work, {
   try {
     await handle.run('map', async () => {
       started = true
+      const telemetry = loaded.sdk.telemetry
       try {
-        value = await work(loaded.sdk.telemetry)
+        value = await work(telemetry)
       } catch (error) {
         workError = error
+        telemetry?.set?.({ errorStage: workError?.stage ?? 'unknown' })
         throw Object.assign(new Error('Action execution failed'), { code: 'ACTION_EXECUTION_FAILED' })
       }
     }, {

src/main.mjs

@@ -18,6 +18,9 @@ const log = message => process.stderr.write(`${message}\n`)
 const notice = message => process.stdout.write(`::notice title=evlog::${message}\n`)
 const fail = message => process.stdout.write(`::error title=evlog::${message}\n`)
 
+/** Stamp the stage an error came from, for telemetry's errorStage field. */
+const tag = (stage, error) => Object.assign(error, { stage })
+
 /** Why a package failed its gate, read off the JSON the CLI printed. */
 function reasons(json, minScore) {
   const out = []
@@ -48,15 +51,25 @@ async function main(inputs, telemetry) {
   const root = resolve(event.workspace, inputs.workingDirectory)
 
   const packages = inputs.packages.length > 0 ? expandPackages(inputs.packages, root) : [{ dir: root, name: '.' }]
-  if (packages.length === 0) throw new InputError(`\`packages\` matched nothing under ${inputs.workingDirectory}`)
+  if (packages.length === 0) throw tag('inputs', new InputError(`\`packages\` matched nothing under ${inputs.workingDirectory}`))
   for (const pkg of packages) {
-    if (!isPackage(pkg.dir)) throw new InputError(`${pkg.name} has no package.json`)
+    if (!isPackage(pkg.dir)) throw tag('inputs', new InputError(`${pkg.name} has no package.json`))
   }
   log(`evlog: ${packages.length} package${packages.length === 1 ? '' : 's'} · ${spec}`)
 
-  const baseline = resolveBaseline(inputs.baseline, event)
+  let baseline
+  try {
+    baseline = resolveBaseline(inputs.baseline, event)
+  } catch (error) {
+    throw tag('inputs', error)
+  }
   telemetry?.set({ packages: packages.length, baselineMode: baseline.mode })
-  const base = baseline.mode === 'base' ? checkoutBase({ workspace: root, ref: baseline.ref, log }) : undefined
+  let base
+  try {
+    base = baseline.mode === 'base' ? checkoutBase({ workspace: root, ref: baseline.ref, log }) : undefined
+  } catch (error) {
+    throw tag('baseline', error)
+  }
   const baselineRef = baseline.mode === 'base' ? baseline.ref : baseline.mode === 'spec' ? baseline.spec : undefined
 
   const results = []
@@ -94,6 +107,8 @@ async function main(inputs, telemetry) {
       })
       log(`evlog: ${pkg.name === '.' ? json.map.projectName : pkg.name} · ${packageLine(results.at(-1), { baselineRef })}`)
     }
+  } catch (error) {
+    throw tag('cli', error)
   } finally {
     base?.cleanup()
   }
@@ -106,16 +121,21 @@ async function main(inputs, telemetry) {
   if (inputs.summary && event.summaryFile) appendFileSync(event.summaryFile, `${renderSummary(results, context)}\n`)
 
   if (inputs.annotations) {
-    const outcome = await createCheckRun({
-      event,
-      token: inputs.token,
-      name: CHECK_NAME,
-      headSha: event.pullRequest?.headSha ?? event.sha,
-      conclusion: outputs.passed ? 'success' : inputs.gate ? 'failure' : 'neutral',
-      title: plain(`${outputs.score} · ${verdict(results, context)}`),
-      summary: report,
-      annotations: results.flatMap(result => findings(result, { baselineRef })),
-    })
+    let outcome
+    try {
+      outcome = await createCheckRun({
+        event,
+        token: inputs.token,
+        name: CHECK_NAME,
+        headSha: event.pullRequest?.headSha ?? event.sha,
+        conclusion: outputs.passed ? 'success' : inputs.gate ? 'failure' : 'neutral',
+        title: plain(`${outputs.score} · ${verdict(results, context)}`),
+        summary: report,
+        annotations: results.flatMap(result => findings(result, { baselineRef })),
+      })
+    } catch (error) {
+      throw tag('check', error)
+    }
     telemetry?.set({ checkOutcome: outcome.outcome })
     if (outcome.outcome === 'created') log(`evlog: check run ${outcome.url} · ${outcome.annotations} annotation${outcome.annotations === 1 ? '' : 's'}`)
     else {
@@ -127,7 +147,12 @@ async function main(inputs, telemetry) {
 
   if (inputs.comment !== 'never' && event.pullRequest) {
     const create = inputs.comment === 'always' || !outputs.passed
-    const outcome = await upsertComment({ event, token: inputs.token, key: inputs.commentKey, body: report, create })
+    let outcome
+    try {
+      outcome = await upsertComment({ event, token: inputs.token, key: inputs.commentKey, body: report, create })
+    } catch (error) {
+      throw tag('comment', error)
+    }
     telemetry?.set({ commentOutcome: outcome.outcome })
     if (outcome.outcome === 'skipped') notice(`pull request comment skipped: ${outcome.reason}`)
     else if (outcome.outcome === 'failed') fail(`pull request comment ${outcome.reason}`)

test/telemetry.test.mjs

@@ -1,11 +1,14 @@
 import assert from 'node:assert/strict'
 import { describe, it } from 'node:test'
-import { COLLECT_FIELDS, runWithTelemetry, scanFields } from '../src/lib/telemetry.mjs'
+import { mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { COLLECT_FIELDS, resolveVersion, runWithTelemetry, scanFields } from '../src/lib/telemetry.mjs'
 
 function setup({ runError, setupError, afterError, cleanupError } = {}) {
-  const calls = { notices: [], cleaned: 0, loaded: 0, options: undefined, runs: [] }
+  const calls = { notices: [], cleaned: 0, loaded: 0, options: undefined, runs: [], sets: [] }
   const handle = {
-    set() {},
+    set(fields) { calls.sets.push(fields) },
     async run(command, work, options) {
       calls.runs.push({ command, options })
       if (runError) throw runError
@@ -131,6 +134,56 @@ describe('runWithTelemetry', () => {
     assert.equal(calls.cleaned, 1)
     assert.deepEqual(calls.notices, [])
   })
+
+  it('reports the stage a work error came from', async () => {
+    const { calls, options } = setup()
+    const error = new Error('git checkout failed')
+    error.stage = 'baseline'
+    await assert.rejects(runWithTelemetry(inputs, async () => { throw error }, options), actual => actual === error)
+    assert.deepEqual(calls.sets, [{ errorStage: 'baseline' }])
+  })
+
+  it('reports an unknown stage when the work error carries none', async () => {
+    const { calls, options } = setup()
+    const error = new Error('something broke')
+    await assert.rejects(runWithTelemetry(inputs, async () => { throw error }, options), actual => actual === error)
+    assert.deepEqual(calls.sets, [{ errorStage: 'unknown' }])
+  })
+})
+
+describe('resolveVersion', () => {
+  it('prefers GITHUB_ACTION_REF', () => {
+    assert.equal(resolveVersion({ GITHUB_ACTION_REF: 'v1' }), 'v1')
+  })
+
+  it('falls back to the package.json the runner checked out at GITHUB_ACTION_PATH', async () => {
+    const directory = await mkdtemp(join(tmpdir(), 'evlog-action-'))
+    await writeFile(join(directory, 'package.json'), JSON.stringify({ name: '@evlog/action', version: '1.3.0' }))
+    try {
+      assert.equal(resolveVersion({ GITHUB_ACTION_PATH: directory }), '1.3.0')
+    } finally {
+      await rm(directory, { recursive: true, force: true })
+    }
+  })
+
+  it('ignores a package.json without a version', async () => {
+    const directory = await mkdtemp(join(tmpdir(), 'evlog-action-'))
+    await writeFile(join(directory, 'package.json'), JSON.stringify({ name: '@evlog/action' }))
+    try {
+      assert.equal(resolveVersion({ GITHUB_ACTION_PATH: directory }), 'local')
+    } finally {
+      await rm(directory, { recursive: true, force: true })
+    }
+  })
+
+  it('reports local when neither the ref nor the checkout is available', async () => {
+    const directory = await mkdtemp(join(tmpdir(), 'evlog-action-'))
+    try {
+      assert.equal(resolveVersion({ GITHUB_ACTION_PATH: directory }), 'local')
+    } finally {
+      await rm(directory, { recursive: true, force: true })
+    }
+  })
 })
 
 describe('scanFields', () => {