Add benchmark job in CI

Author: natecook1000

.github/workflows/pull_request.yml

@@ -19,3 +19,61 @@ jobs:
       license_header_check_project_name: "Swift.org"
       unacceptable_language_check_enabled: false
       format_check_enabled: false
+  bench:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    env:
+      BUILD_CMD: swift build -c release
+      BENCH_CMD: .build/release/RegexBenchmark
+      BASELINE_FILE: benchmark-baseline
+      COMPARE_FILE: benchmark-pr
+      COMPARE_OUT_FILE: benchmark-results.txt
+    steps:
+      - name: Check out baseline branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+          fetch-depth: 0
+      - name: Build baseline branch
+        working-directory: base
+        run: |
+          set -euo pipefail
+          eval "$BUILD_CMD"
+      - name: Run baseline benchmark
+        working-directory: base
+        run: |
+          set -euo pipefail
+          eval "$BENCH_CMD --save $RUNNER_TEMP/$BASELINE_FILE"
+          test -s "$RUNNER_TEMP/$BASELINE_FILE" || { echo "Baseline not created at $BASELINE_FILE"; exit 1; }
+      - name: Check out PR branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: pr
+          fetch-depth: 0
+      - name: Build PR branch
+        working-directory: pr
+        run: |
+          set -euo pipefail
+          eval "$BUILD_CMD"
+      - name: Run PR benchmark
+        working-directory: pr
+        run: |
+          set -euo pipefail
+          eval "$BENCH_CMD --save $RUNNER_TEMP/$COMPARE_FILE"
+          test -s "$RUNNER_TEMP/$COMPARE_FILE" || { echo "Comparison not created at $COMPARE_FILE"; exit 1; }
+          eval "$BENCH_CMD --compare $RUNNER_TEMP/$BASELINE_FILE" | tee "$RUNNER_TEMP/$COMPARE_OUT_FILE"
+      - name: Compare benchmarks
+        working-directory: pr
+        run: |
+          set -euo pipefail
+          eval "$BENCH_CMD --load $RUNNER_TEMP/$COMPARE_FILE --compare $RUNNER_TEMP/$BASELINE_FILE " | tee "$RUNNER_TEMP/$COMPARE_OUT_FILE"
+      - name: Upload benchmark artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: |
+            ${{ runner.temp }}/${{ env.BASELINE_FILE }}
+            ${{ runner.temp }}/${{ env.COMPARE_FILE }}
+            ${{ runner.temp }}/${{ env.COMPARE_OUT_FILE }}

Sources/RegexBenchmark/Utils/Stats.swift

@@ -15,7 +15,7 @@ enum Stats {}
 
 extension Stats {
   // Maximum allowed standard deviation is 7.5% of the median runtime
-  static let maxAllowedStdev = 0.075
+  static let maxAllowedStdev = 0.15
 
   static func tTest(_ a: Measurement, _ b: Measurement) -> Bool {
     // Student's t-test