benchmarks.yaml

name: Benchmarks

on:
  workflow_dispatch: {}
  push:
    branches: [main]

permissions:
  contents: read

jobs:
    # Check what types of changes this PR contains
  check-changes:
    name: Check what files changed
    runs-on: ubuntu-24.04
    outputs:
      go: ${{ steps.changes.outputs.go }}
      bench: ${{ steps.changes.outputs.bench }}
    steps:
      - name: Check out code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0
          persist-credentials: false

      - name: Download OPA
        uses: open-policy-agent/setup-opa@b2b258e089860efaadaaf71bf6e3aecb4a3eeff1 # v2.4.0
        with:
          version: edge

      - name: Check for file changes
        id: changes
        env:
          BEFORE_SHA: ${{ github.event.before }}
          CURRENT_SHA: ${{ github.event.after }}
        run: |
          set -e

          # Default to running all checks
          echo "go=true" >> $GITHUB_OUTPUT

          echo "Comparing $BEFORE_SHA with $CURRENT_SHA"
          git diff --name-only "$BEFORE_SHA" "$CURRENT_SHA" \
            | jq -R '{filename: .}' | jq -s '.' > changed_files.json

          if [ ! -s changed_files.json ] || [ "$(cat changed_files.json)" = "[]" ]; then
            echo "Warning: No changed files found"
            exit 0
          fi

          echo "Changed files:"
          jq -r '.[].filename' changed_files.json

          opa eval \
          --data build/policy/pr-check/pr_check.rego \
          --input changed_files.json \
          --format pretty \
          'data.policy["pr-check"]' > opa_result.json

          go_result=$(jq -r '.changes.go // false' opa_result.json)
          bench_result=$(jq -c '.changes.bench // []' opa_result.json)

          echo "go=${go_result}" >> $GITHUB_OUTPUT
          echo "bench=${bench_result}" >> $GITHUB_OUTPUT

          echo "Final outputs:"
          echo "  go=${go_result}"
          echo "  bench=${bench_result}"

  benchmarks:
    permissions:
      contents: write # we'll push to the `benchmarks` branch
    name: Benchmarks
    needs: check-changes
    if: ${{ needs.check-changes.outputs.go == 'true' }}
    uses: ./.github/workflows/run-benchmarks.yaml
    with:
      publish: true
      publish_branch: benchmarks

  regression-check:
    permissions:
      contents: read
      pull-requests: write
    name: Check for regressions
    runs-on: ubuntu-24.04
    needs: [check-changes]
    if: ${{ needs.check-changes.outputs.bench != '' && needs.check-changes.outputs.bench != '[]' }}
    steps:
      - name: Check out code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 1
          persist-credentials: false
      - name: Fetch base commit
        env:
          BEFORE_SHA: ${{ github.event.before }}
        run: git fetch --depth=1 origin "$BEFORE_SHA"
      - id: go_version
        name: Read go version
        run: echo "go_version=$(cat .go-version)" >> $GITHUB_OUTPUT
      - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version: ${{ steps.go_version.outputs.go_version }}
      - name: Install tools
        run: cd build/tools && go install tool
      - name: Run benchmarks
        env:
          BEFORE_SHA: ${{ github.event.before }}
          AFTER_SHA: ${{ github.event.after }}
          BENCH_PKGS: ${{ needs.check-changes.outputs.bench }}
        run: |
          for pkg in $(echo "$BENCH_PKGS" | jq -r '.[]'); do
            benchlab \
              -commit "$BEFORE_SHA","$AFTER_SHA" \
              -pkg "$pkg" \
              -host local:tags=opa_wasm \
              -reps 3 \
              -benchtime 300ms \
              -run '^$'
          done
      - name: Comment on PR with benchmark results
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          COMMIT_SHA: ${{ github.sha }}
        run: |
          PR_NUMBER=$(gh pr list --search "${COMMIT_SHA}" --state merged --json number --jq '.[0].number')

          if [ -z "$PR_NUMBER" ] || [ "$PR_NUMBER" = "null" ]; then
            echo "Could not find originating PR for commit ${COMMIT_SHA}"
            exit 0
          fi

          {
            echo "## Benchmark Comparison (${BEFORE_SHA} vs ${AFTER_SHA})"
            echo ""
            echo '```'
            # Each benchstat file ends with a compact benchmark × host delta table;
            # extract from that header line onward so the comment stays readable.
            for f in .benchlab/benchstat.*.txt; do
              awk '/^benchmark \\ host/{found=1} found{print}' "$f"
              echo ""
            done
            echo '```'
            echo ""
            echo "_This comment was automatically generated by the benchmarks workflow._"
          } > body.md

          gh pr comment "${PR_NUMBER}" --body-file body.md

  notebook:
    permissions:
      contents: write # we'll push to the `benchmarks` branch
    name: update notebook
    runs-on: ubuntu-24.04
    needs: [check-changes, benchmarks] # force sequential commits for notebook and benchmark results
    if: ${{ needs.check-changes.outputs.go == 'true' }}
    steps:
      - uses: open-policy-agent/setup-opa@b2b258e089860efaadaaf71bf6e3aecb4a3eeff1 # v2.4.0
      - name: Check out code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: benchmarks
          persist-credentials: true
      - name: Setup Java
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
        with:
          distribution: 'temurin'
          java-version: '21'
      - name: Setup Clojure
        uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4.0.1
        with:
          install: true
          cache: true
          mise_toml: |
            [tools]
            clojure = "1.12.5.1638"
      - name: Cache Clojure dependencies
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
        with:
          path: |
            ~/.m2/repository
            ~/.gitlibs
          key: clj-${{ hashFiles('clay/deps.edn') }}
          restore-keys: clj-
      - name: Clean previous output
        run: rm -rf docs/*.html
      - name: update notebook
        run: |
          clojure -J-Xss32m -M -m opa-bench.generate
        working-directory: clay/
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: commit if changed
        working-directory: docs/
        run: |
          if ! git diff-index --quiet HEAD -- .; then
            git config --local user.email "${GITHUB_ACTOR}@users.noreply.github.com"
            git config --local user.name "${GITHUB_ACTOR}"
            git add -f .
            git diff --staged --name-only
            git commit -m "benchmarks: update notebook for ${GITHUB_SHA}"
            git push origin benchmarks
          else
            echo "no changes, no commit"
          fi