benchmark.yaml

name: Package Manager Benchmarks
on:
  push:
  workflow_dispatch:
    inputs:
      fixtures:
        description: "The fixture to run the benchmarks on"
        default: '["next", "astro", "vue", "svelte", "large", "babylon"]'
      variations:
        description: "The benchmark variations to run"
        default: '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules", "registry-clean", "registry-lockfile"]'
      binaries:
        description: "The binaries to run the benchmarks on"
        default: '"npm,yarn,berry,zpm,pnpm,vlt,bun,deno,nx,turbo,node"'
      warmup:
        description: "The number of warmup runs on each benchmark"
        default: "2"
      runs:
        description: "The number of runs on each benchmark"
        default: "10"
  schedule:
    # GitHub Actions cron uses UTC. 10:17 UTC ~= 2:17 AM Pacific (PST).
    - cron: "17 10 * * *"

# Prevent multiple runs from interfering with each other
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  benchmark:
    name: "Run Benchmarks"
    runs-on: ubuntu-latest
    timeout-minutes: 120
    strategy:
      matrix:
        fixture: ${{ fromJson((github.event_name == 'push' && github.ref != 'refs/heads/main') && '["next", "astro", "vue", "svelte"]' || (inputs.fixtures || '["next", "astro", "vue", "svelte", "large", "babylon"]')) }}
        variation: ${{ fromJson(inputs.variations || '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules", "registry-clean", "registry-lockfile"]') }}
        include:
          - variation: "run"
            fixture: "run"
    permissions:
      contents: read
      id-token: write
    env:
      BENCH_INCLUDE: ${{ fromJson(inputs.binaries || '"npm,yarn,berry,zpm,pnpm,vlt,bun,deno,nx,turbo,node"') }}
      BENCH_WARMUP: ${{ (github.event_name == 'push' && github.ref != 'refs/heads/main') && '1' || (inputs.warmup || '2') }}
      BENCH_RUNS: ${{ (github.event_name == 'push' && github.ref != 'refs/heads/main') && '1' || (inputs.runs || '10') }}
    steps:
      - uses: actions/checkout@v6
      - name: Install Node
        uses: actions/setup-node@v6
        with:
          node-version: '24'
          package-manager-cache: false
      - name: Install & Setup Tools
        run: |
          bash ./scripts/setup.sh
      - name: Configure AWS Credentials
        if: startsWith(matrix.variation, 'registry-')
        uses: aws-actions/configure-aws-credentials@v6
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1
      - name: Get CodeArtifact Token
        if: startsWith(matrix.variation, 'registry-')
        id: aws
        run: |
          CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \
            --domain vlt \
            --domain-owner 451504312483 \
            --region us-east-1 \
            --query authorizationToken \
            --output text)
          echo "::add-mask::$CODEARTIFACT_AUTH_TOKEN"
          echo "token=$CODEARTIFACT_AUTH_TOKEN" >> "$GITHUB_OUTPUT"
      - name: Run Benchmarks variations
        env:
          CODEARTIFACT_AUTH_TOKEN: ${{ steps.aws.outputs.token }}
        run: |
          ./bench run \
            --fixtures="${{ matrix.fixture }}" \
            --variation="${{ matrix.variation }}"
      - name: Upload Benchmark Results
        uses: actions/upload-artifact@v7
        with:
          name: results-${{ matrix.fixture }}-${{ matrix.variation }}
          path: ./results/${{ matrix.fixture }}/${{ matrix.variation }}/
          retention-days: 7
      - name: Upload Versions Info
        uses: actions/upload-artifact@v7
        with:
          name: versions-${{ matrix.fixture }}-${{ matrix.variation }}
          path: ./results/versions.json
          retention-days: 7
  process:
    name: "Process Results"
    runs-on: ubuntu-latest
    needs: [benchmark]
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v6
      - name: Install Node
        uses: actions/setup-node@v6
        with:
          node-version: "24"
      - name: Download Results
        uses: actions/download-artifact@v8
        with:
          path: results
          pattern: results-*
      - name: Download Versions
        uses: actions/download-artifact@v8
        with:
          path: versions-temp
          pattern: versions-*
      - name: Process Results
        run: |
          ./bench process
      - name: Install vlt
        run: |
          npm install -g vlt@latest
      - name: Build Charts View
        run: |
          pushd app
          vlt install || true
          vlt run build
          popd
      - name: Upload Processed Results
        uses: actions/upload-artifact@v7
        with:
          name: results
          path: results/
          retention-days: 7
  deploy:
    name: "Deploy Results"
    runs-on: ubuntu-latest
    needs: [process]
    permissions:
      contents: write
    if: github.ref == 'refs/heads/main'
    steps:
      - uses: actions/checkout@v6
      - name: Download Results
        uses: actions/download-artifact@v8
        with:
          name: results
          path: results/
      - name: Deploy to GitHub Pages
        uses: peaceiris/actions-gh-pages@v4
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: results
          keep_files: true