|
| 1 | +name: 'Harness Score' |
| 2 | +description: >- |
| 3 | + Deterministic AI-harness maturity audit (Cursor-first): scans the repository, |
| 4 | + reports a maturity level (L0–L4), optionally fails the build below a minimum |
| 5 | + level, and emits an SVG badge and markdown report. |
| 6 | +author: 'Fernando Paladini' |
| 7 | +branding: |
| 8 | + icon: 'shield' |
| 9 | + color: 'green' |
| 10 | + |
| 11 | +inputs: |
| 12 | + min-level: |
| 13 | + description: 'Fail the job when the maturity level is below this (0–4). Use 0 to never fail.' |
| 14 | + required: false |
| 15 | + default: '0' |
| 16 | + badge: |
| 17 | + description: 'Path to write the SVG maturity badge (empty to skip).' |
| 18 | + required: false |
| 19 | + default: 'harness-badge.svg' |
| 20 | + report: |
| 21 | + description: 'Path to write the markdown report (empty to skip).' |
| 22 | + required: false |
| 23 | + default: '' |
| 24 | + working-directory: |
| 25 | + description: 'Directory to scan.' |
| 26 | + required: false |
| 27 | + default: '.' |
| 28 | + version: |
| 29 | + description: 'harness-score npm version to run.' |
| 30 | + required: false |
| 31 | + default: 'latest' |
| 32 | + comment: |
| 33 | + description: >- |
| 34 | + Post (or update) a sticky pull-request comment showing the harness |
| 35 | + score and, on pull_request events, its delta against the base branch. |
| 36 | + Opt-in: the calling workflow must grant `pull-requests: write`. |
| 37 | + required: false |
| 38 | + default: 'false' |
| 39 | + include-user-harness: |
| 40 | + description: 'Include user-level harness artifacts (~/.cursor, ~/.claude, etc.) in the effective score.' |
| 41 | + required: false |
| 42 | + default: 'false' |
| 43 | + include-system-harness: |
| 44 | + description: 'Include system-level harness artifacts in the effective score (minimal support in v1).' |
| 45 | + required: false |
| 46 | + default: 'false' |
| 47 | + gate: |
| 48 | + description: 'Which score --min-level compares against: maturity (repo-only, default) or effective.' |
| 49 | + required: false |
| 50 | + default: 'maturity' |
| 51 | + config: |
| 52 | + description: 'Path to a harness-score JSON config file (default: .harness-score.json in the scan root).' |
| 53 | + required: false |
| 54 | + default: '' |
| 55 | + |
| 56 | +outputs: |
| 57 | + level: |
| 58 | + description: 'Maturity level index (0–4).' |
| 59 | + value: ${{ steps.scan.outputs.level }} |
| 60 | + level-name: |
| 61 | + description: 'Maturity level name (Unharnessed … Self-correcting).' |
| 62 | + value: ${{ steps.scan.outputs.level-name }} |
| 63 | + percent: |
| 64 | + description: 'Maturity score percentage.' |
| 65 | + value: ${{ steps.scan.outputs.percent }} |
| 66 | + effective-level: |
| 67 | + description: 'Effective level index when global scopes are enabled.' |
| 68 | + value: ${{ steps.scan.outputs.effective-level }} |
| 69 | + effective-percent: |
| 70 | + description: 'Effective score percentage when global scopes are enabled.' |
| 71 | + value: ${{ steps.scan.outputs.effective-percent }} |
| 72 | + |
| 73 | +runs: |
| 74 | + using: 'composite' |
| 75 | + steps: |
| 76 | + - id: scan |
| 77 | + shell: bash |
| 78 | + working-directory: ${{ inputs.working-directory }} |
| 79 | + run: | |
| 80 | + set -euo pipefail |
| 81 | + ARGS=(--quiet --json) |
| 82 | + [ -n "${{ inputs.badge }}" ] && ARGS+=(--badge "${{ inputs.badge }}") |
| 83 | + [ -n "${{ inputs.report }}" ] && ARGS+=(--md "${{ inputs.report }}") |
| 84 | + [ -n "${{ inputs.config }}" ] && ARGS+=(--config "${{ inputs.config }}") |
| 85 | + [ "${{ inputs.include-user-harness }}" = "true" ] && ARGS+=(--scope user) |
| 86 | + [ "${{ inputs.include-system-harness }}" = "true" ] && ARGS+=(--scope system) |
| 87 | + [ -n "${{ inputs.gate }}" ] && ARGS+=(--gate "${{ inputs.gate }}") |
| 88 | + npx -y "harness-score@${{ inputs.version }}" . "${ARGS[@]}" > harness-report.json |
| 89 | +
|
| 90 | + LEVEL=$(node -p "JSON.parse(require('fs').readFileSync('harness-report.json','utf8')).level.index") |
| 91 | + NAME=$(node -p "JSON.parse(require('fs').readFileSync('harness-report.json','utf8')).level.name") |
| 92 | + PERCENT=$(node -p "JSON.parse(require('fs').readFileSync('harness-report.json','utf8')).score.percent") |
| 93 | + EFF_LEVEL=$(node -p "JSON.parse(require('fs').readFileSync('harness-report.json','utf8')).effective.level.index") |
| 94 | + EFF_PERCENT=$(node -p "JSON.parse(require('fs').readFileSync('harness-report.json','utf8')).effective.score.percent") |
| 95 | + GATE=$(node -p "JSON.parse(require('fs').readFileSync('harness-report.json','utf8')).gate") |
| 96 | + GATE_LEVEL=$(node -p "const r=JSON.parse(require('fs').readFileSync('harness-report.json','utf8')); (r.gate==='effective'?r.effective:r).level.index") |
| 97 | + GATE_GAPS=$(node -p "const r=JSON.parse(require('fs').readFileSync('harness-report.json','utf8')); (r.gate==='effective'?r.effective:r).level.nextLevelGaps.join('; ')") |
| 98 | + echo "level=$LEVEL" >> "$GITHUB_OUTPUT" |
| 99 | + echo "level-name=$NAME" >> "$GITHUB_OUTPUT" |
| 100 | + echo "percent=$PERCENT" >> "$GITHUB_OUTPUT" |
| 101 | + echo "effective-level=$EFF_LEVEL" >> "$GITHUB_OUTPUT" |
| 102 | + echo "effective-percent=$EFF_PERCENT" >> "$GITHUB_OUTPUT" |
| 103 | + echo "## Harness Score: L$LEVEL · $NAME ($PERCENT% maturity)" >> "$GITHUB_STEP_SUMMARY" |
| 104 | + if [ "$EFF_LEVEL" != "$LEVEL" ] || [ "$EFF_PERCENT" != "$PERCENT" ]; then |
| 105 | + echo "Effective: L$EFF_LEVEL ($EFF_PERCENT%) · gate: $GATE" >> "$GITHUB_STEP_SUMMARY" |
| 106 | + fi |
| 107 | + node -e " |
| 108 | + const r = JSON.parse(require('fs').readFileSync('harness-report.json','utf8')); |
| 109 | + const rows = r.dimensions.map(d => '| ' + d.title + ' | ' + d.earned + '/' + d.max + ' | ' + d.percent + '% |').join('\n'); |
| 110 | + require('fs').appendFileSync(process.env.GITHUB_STEP_SUMMARY, '\n| Dimension | Score | % |\n|---|---|---|\n' + rows + '\n'); |
| 111 | + " |
| 112 | +
|
| 113 | + if [ "$GATE_LEVEL" -lt "${{ inputs.min-level }}" ]; then |
| 114 | + echo "::error::Harness ${GATE} L${GATE_LEVEL} is below required L${{ inputs.min-level }}." |
| 115 | + echo "$GATE_GAPS" |
| 116 | + exit 1 |
| 117 | + fi |
| 118 | +
|
| 119 | + - id: baseline |
| 120 | + if: ${{ inputs.comment == 'true' && github.event_name == 'pull_request' }} |
| 121 | + shell: bash |
| 122 | + working-directory: ${{ inputs.working-directory }} |
| 123 | + run: | |
| 124 | + set -euo pipefail |
| 125 | + git fetch origin "${{ github.event.pull_request.base.ref }}" |
| 126 | + git worktree remove /tmp/harness-score-base --force 2>/dev/null || true |
| 127 | + git worktree add /tmp/harness-score-base FETCH_HEAD |
| 128 | + npx -y "harness-score@${{ inputs.version }}" "/tmp/harness-score-base/${{ inputs.working-directory }}" \ |
| 129 | + --quiet --json > /tmp/harness-score-base.json |
| 130 | + git worktree remove /tmp/harness-score-base --force |
| 131 | +
|
| 132 | + - id: diff |
| 133 | + if: ${{ inputs.comment == 'true' && github.event_name == 'pull_request' }} |
| 134 | + shell: bash |
| 135 | + working-directory: ${{ inputs.working-directory }} |
| 136 | + run: | |
| 137 | + set -euo pipefail |
| 138 | + npx -y "harness-score@${{ inputs.version }}" . --quiet --json \ |
| 139 | + --diff /tmp/harness-score-base.json > /tmp/harness-score-diff.json |
| 140 | +
|
| 141 | + - id: comment |
| 142 | + if: ${{ inputs.comment == 'true' && github.event_name == 'pull_request' }} |
| 143 | + uses: actions/github-script@v7 |
| 144 | + with: |
| 145 | + script: | |
| 146 | + const fs = require('fs'); |
| 147 | + const { diff } = JSON.parse(fs.readFileSync('/tmp/harness-score-diff.json', 'utf8')); |
| 148 | + const marker = '<!-- harness-score-report -->'; |
| 149 | + const arrow = diff.level.delta > 0 ? '⬆️' : diff.level.delta < 0 ? '⬇️' : '➡️'; |
| 150 | + const sign = (n) => (n > 0 ? `+${n}` : `${n}`); |
| 151 | +
|
| 152 | + const modelWarning = diff.maturityModelChanged |
| 153 | + ? '> ⚠ Baseline is from a different tool version or maturity model total — some deltas below may reflect that, not repository changes.\n\n' |
| 154 | + : ''; |
| 155 | +
|
| 156 | + const movedDims = diff.dimensions.filter((d) => d.delta !== 0); |
| 157 | + const dimTable = movedDims.length |
| 158 | + ? [ |
| 159 | + '| Dimension | Before | After | Δ |', |
| 160 | + '|---|---|---|---|', |
| 161 | + ...movedDims.map((d) => `| ${d.title} | ${d.before}% | ${d.after}% | ${sign(d.delta)}pp |`), |
| 162 | + ].join('\n') |
| 163 | + : '_No dimension moved._'; |
| 164 | +
|
| 165 | + const gained = diff.checksChanged.filter((c) => c.change === 'newly-passing').map((c) => c.id); |
| 166 | + const lost = diff.checksChanged.filter((c) => c.change === 'newly-failing').map((c) => c.id); |
| 167 | +
|
| 168 | + const body = [ |
| 169 | + marker, |
| 170 | + `### ${arrow} Harness Score: L${diff.level.before} · ${diff.level.beforeName} → L${diff.level.after} · ${diff.level.afterName}`, |
| 171 | + '', |
| 172 | + modelWarning + `**Score:** ${diff.score.before.earned}/${diff.score.before.max} (${diff.score.before.percent}%) → ` + |
| 173 | + `${diff.score.after.earned}/${diff.score.after.max} (${diff.score.after.percent}%) (${sign(diff.score.deltaPercent)}pp)`, |
| 174 | + '', |
| 175 | + dimTable, |
| 176 | + '', |
| 177 | + gained.length ? `**Newly passing:** ${gained.join(', ')}` : '', |
| 178 | + lost.length ? `**Newly failing:** ${lost.join(', ')}` : '', |
| 179 | + '', |
| 180 | + '[Harness Score guide](https://paladini.github.io/harness-score/)', |
| 181 | + ] |
| 182 | + .filter((line) => line !== '') |
| 183 | + .join('\n'); |
| 184 | +
|
| 185 | + const { owner, repo } = context.repo; |
| 186 | + const issue_number = context.issue.number; |
| 187 | + const comments = await github.paginate(github.rest.issues.listComments, { |
| 188 | + owner, |
| 189 | + repo, |
| 190 | + issue_number, |
| 191 | + per_page: 100, |
| 192 | + }); |
| 193 | + const existing = comments.find((c) => c.body && c.body.includes(marker)); |
| 194 | +
|
| 195 | + if (existing) { |
| 196 | + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); |
| 197 | + } else { |
| 198 | + await github.rest.issues.createComment({ owner, repo, issue_number, body }); |
| 199 | + } |
0 commit comments