advisory check for published tarball drift [agent-managed]

.github/workflows/changeset.yaml

@@ -38,20 +38,80 @@ jobs:
           published_packages: ${{ steps.changesets.outputs.publishedPackages }}
           buildkite_webhook_url: ${{ secrets.ENCHIRIDION_BUILDKITE_WEBHOOK_URL }}
 
-  notify-slack:
+  notify-slack-success:
     runs-on: ubuntu-latest
     timeout-minutes: 5
-    needs:
-      version
-      # We only trigger this flow if the publish is happening, inferred by not having a changeset.
-    if: needs.version.outputs.hasChangesets == 'false'
-    env:
-      PUBLISHED: ${{ needs.version.outputs.hasPublished }}
+    needs: version
+    if: needs.version.outputs.hasPublished == 'true'
+    steps:
+      - name: Notify successful publish
+        uses: slackapi/slack-github-action@v1.24.0
+        with:
+          channel-id: 'C02NUQ27G56'
+          slack-message: 'Packages have been published'
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+
+  notify-slack-failure:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    needs: version
+    if: ${{ always() && needs.version.result == 'failure' }}
     steps:
-      - name: Send
+      - uses: actions/checkout@v4
+
+      - name: Get failure context
+        id: context
+        run: |
+          SHA="${{ github.sha }}"
+          SHORT_SHA="${SHA:0:7}"
+          # JSON-escape: an unescaped quote or backslash would break the Block Kit payload.
+          SUBJECT=$(git log -1 --format='%s' "$SHA" | jq -Rj 'tojson | .[1:-1]')
+          echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT"
+          echo "subject=$SUBJECT" >> "$GITHUB_OUTPUT"
+          echo "run_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> "$GITHUB_OUTPUT"
+
+      - name: Notify publish failure
         uses: slackapi/slack-github-action@v1.24.0
         with:
           channel-id: 'C02NUQ27G56'
-          slack-message: ${{ env.PUBLISHED == 'true' && 'Packages have been published' || 'Publishing failed' }}
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": ":x: *Publishing failed*"
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Commit:* <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|`${{ steps.context.outputs.short_sha }}`> ${{ steps.context.outputs.subject }}"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*Pushed by:* ${{ github.actor }}"
+                    }
+                  ]
+                },
+                {
+                  "type": "actions",
+                  "elements": [
+                    {
+                      "type": "button",
+                      "text": {
+                        "type": "plain_text",
+                        "text": "View failed run"
+                      },
+                      "url": "${{ steps.context.outputs.run_url }}"
+                    }
+                  ]
+                }
+              ]
+            }
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/check-package-output.yml

@@ -0,0 +1,27 @@
+name: Check package output
+
+# The inverse of changeset.yaml: that publishes from main and release branches,
+# this advises on feature branches, where a changeset can still be added.
+on:
+  push:
+    branches-ignore:
+      - 'main'
+      - 'release/**'
+      - 'changeset-release/**'
+
+jobs:
+  check-package-output:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: ./.github/actions/setup
+
+      - name: Build workspace
+        run: pnpm turbo build
+
+      - name: Check package output drift
+        run: pnpm check:package-output

.prettierignore

@@ -17,6 +17,7 @@
 !*.scss
 
 # Explicitly ignore...
+.devbox/
 devbox.lock
 pnpm-lock.yaml
 **/CHANGELOG.md

eslint.config.js

@@ -14,6 +14,7 @@ import prettierRules from './.prettierrc.js'
 export default tseslint.config(
   {
     ignores: [
+      '.devbox/**',
       '**/CHANGELOG.md',
       '**/tmp/**',
       '**/*.d.ts',

package.json

@@ -25,6 +25,7 @@
     "reset": "pnpm clean && pnpm install --force",
     "chromatic": "chromatic",
     "changeset": "changeset",
+    "check:package-output": "tsx scripts/check-package-output.ts",
     "plop": "plop",
     "pkg:aio": "pnpm -F @kaizen/components",
     "pkg:tokens": "pnpm -F @kaizen/design-tokens",
@@ -82,6 +83,7 @@
     "stylelint": "^17.14.1",
     "stylelint-config-standard": "^40.0.0",
     "stylelint-config-standard-scss": "^17.0.0",
+    "tsx": "^4.23.11",
     "turbo": "^2.10.9",
     "typescript": "^5.9.3",
     "typescript-eslint": "^8.66.0",

scripts/check-package-output.ts

@@ -0,0 +1,121 @@
+/**
+ * Advisory: does the tarball we'd publish differ from the one already on npm?
+ *
+ * Assumes `pnpm turbo build` has already run. Exits 1 if any package's output
+ * differs with no changeset covering it — a human decides whether that warrants
+ * a release.
+ */
+
+/* eslint-disable no-console */
+
+import { execFileSync } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+const run = (cmd: string, args: string[], cwd?: string): string =>
+  execFileSync(cmd, args, { encoding: 'utf-8', cwd, maxBuffer: 64 * 1024 * 1024 }).trim()
+
+const sha = (data: Buffer | string): string => createHash('sha256').update(data).digest('hex')
+
+const fingerprint = (dir: string): string => {
+  const files = readdirSync(dir, { recursive: true, withFileTypes: true })
+    .filter((entry) => entry.isFile())
+    .map((entry) => join(entry.parentPath, entry.name).slice(dir.length + 1))
+    .sort()
+  return sha(files.map((f) => `${sha(readFileSync(join(dir, f)))}  ${f}`).join('\n'))
+}
+
+/** Extract into a directory of its own, so the tarball file never lands in the fingerprint. */
+const extractedFingerprint = (tarball: string, dest: string): string => {
+  const contents = join(dest, 'contents')
+  mkdirSync(contents)
+  run('tar', ['-xf', tarball, '-C', contents, '--strip-components=1'])
+  return fingerprint(contents)
+}
+
+const packedFingerprint = (pkgDir: string, tmp: string): string => {
+  const dest = mkdtempSync(join(tmp, 'local-'))
+  const tarball = run('pnpm', ['pack', '--pack-destination', dest], pkgDir).split('\n').pop()!
+  return extractedFingerprint(tarball, dest)
+}
+
+/** Null means this version was never published, which is a skip rather than a failure. */
+const publishedFingerprint = async (
+  name: string,
+  version: string,
+  tmp: string,
+): Promise<string | null> => {
+  const metadata = await fetch(`https://registry.npmjs.org/${name}/${version}`)
+  if (metadata.status === 404) return null
+  if (!metadata.ok) throw new Error(`registry returned ${metadata.status} for ${name}@${version}`)
+
+  const { dist } = (await metadata.json()) as { dist: { tarball: string } }
+  const download = await fetch(dist.tarball)
+  if (!download.ok) throw new Error(`tarball download failed for ${name}@${version}`)
+
+  const dest = mkdtempSync(join(tmp, 'published-'))
+  const tarball = join(dest, 'published.tgz')
+  writeFileSync(tarball, Buffer.from(await download.arrayBuffer()))
+  return extractedFingerprint(tarball, dest)
+}
+
+const releasedPackages = (): Set<string> => {
+  const out = join(mkdtempSync(join(tmpdir(), 'changeset-')), 'status.json')
+  try {
+    execFileSync('pnpm', ['changeset', 'status', '--since=origin/main', `--output=${out}`], {
+      stdio: 'ignore',
+    })
+  } catch {
+    // Non-zero simply means uncovered changes; the status file is still written.
+  }
+  try {
+    const status = JSON.parse(readFileSync(out, 'utf-8'))
+    return new Set<string>(status.releases.map((r: { name: string }) => r.name))
+  } catch {
+    return new Set()
+  }
+}
+
+const root = run('git', ['rev-parse', '--show-toplevel'])
+const tmp = mkdtempSync(join(tmpdir(), 'package-output-'))
+const released = releasedPackages()
+
+const packages = readdirSync(join(root, 'packages'), { withFileTypes: true })
+  .filter((entry) => entry.isDirectory())
+  .map((entry) => join(root, 'packages', entry.name))
+  .flatMap((dir) => {
+    try {
+      const pkg = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf-8'))
+      return pkg.private === true ? [] : [{ dir, name: pkg.name, version: pkg.version }]
+    } catch {
+      return []
+    }
+  })
+
+const drifted: string[] = []
+
+for (const { dir, name, version } of packages) {
+  const published = await publishedFingerprint(name, version, tmp)
+  if (published === null) {
+    console.log(`? ${name} — ${version} not on registry, skipped`)
+    continue
+  }
+  if (packedFingerprint(dir, tmp) === published) {
+    console.log(`= ${name} — matches ${version}`)
+  } else if (released.has(name)) {
+    console.log(`~ ${name} — differs, changeset present`)
+  } else {
+    console.log(`! ${name} — differs from ${version}, no changeset`)
+    drifted.push(name)
+  }
+}
+
+if (drifted.length > 0) {
+  console.log(
+    `\nOutput differs from the published version with no changeset: ${drifted.join(', ')}.\n` +
+      `Advisory only — add a changeset if these changes should ship.`,
+  )
+  process.exit(1)
+}