Workflow for ASV Benchmarks in PR

Author: lapp0

.github/workflows/asv_benchmark_pr.yml

@@ -0,0 +1,72 @@
+name: Benchmark PR
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read  # Read access for repository contents
+  pull-requests: write  # Write access for pull requests
+
+env:
+  PYTHON_VERSION: "3.10"
+  WORKING_DIR: ${{ github.workspace }}/benchmarks
+
+jobs:
+  benchmark-pr:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: ${{ env.WORKING_DIR }}
+
+    steps:
+
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install asv virtualenv lf-asv-formatter
+
+    - name: Create ASV machine config file
+      run: asv machine --machine gh-runner --yes
+
+    - name: Save comparison of PR against main branch
+      run: |
+        # prepare main branch for comparison
+        git remote add upstream https://github.com/${{ github.repository }}.git
+        git fetch upstream main
+
+        # Run benchmarks, writing comment contents to ./output
+        asv continuous upstream/main HEAD \
+            --factor 1.1 --sort ratio --split --interleave-rounds -a repeat=7
+        asv compare upstream/main HEAD --factor 1.1 --sort ratio --split | tee output
+        python -m lf_asv_formatter --asv_version "$(echo asv --version)"
+        printf "Benchmark Suite Results:\n\n" >> comment_body
+        cat output >> comment_body
+
+    # from https://github.com/hombit/load_ztfdr_for_tape/blob/9acf7c83/.github/workflows/asv-pr.yml
+    - name: Find benchmarks comment
+      uses: peter-evans/find-comment@v2
+      id: find-comment
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        comment-author: 'github-actions[bot]'
+        body-includes: Benchmark Suite Results
+
+    - name: Create or update benchmarks comment
+      uses: peter-evans/create-or-update-comment@v3
+      with:
+        comment-id: ${{ steps.find-comment.outputs.comment-id }}
+        issue-number: ${{ github.event.pull_request.number }}
+        body-path: ${{ env.WORKING_DIR }}/comment_body
+        edit-mode: replace

benchmarks/__init__.py

@@ -0,0 +1,19 @@
+{
+    "version": 1,
+    "project": "Outlines",
+    "project_url": "https://outlines-dev.github.io/outlines/",
+    "repo": "..",
+    "branches": [
+	"HEAD"
+    ],
+    "build_command": [
+	"python -m build -Cbuilddir=builddir --wheel --outdir {build_cache_dir} {build_dir}"
+    ],
+    "environment_type": "virtualenv",
+    "show_commit_url": "https://github.com/lapp0/outlines/commit/",
+    "benchmark_dir": ".",
+    "env_dir": "env",
+    "results_dir": "results",
+    "html_dir": "html",
+    "build_cache_size": 8
+}

benchmarks/asv.conf.json

@@ -0,0 +1,32 @@
+# Write the benchmarking functions here.
+# See "Writing benchmarks" in the asv docs for more information.
+
+
+class TimeSuite:
+    """
+    An example benchmark that times the performance of various kinds
+    of iterating over dictionaries in Python.
+    """
+
+    def setup(self):
+        self.d = {}
+        for x in range(500):
+            self.d[x] = None
+
+    def time_keys(self):
+        for key in self.d.keys():
+            pass
+
+    def time_values(self):
+        for value in self.d.values():
+            pass
+
+    def time_range(self):
+        d = self.d
+        for key in range(500):
+            d[key]
+
+
+class MemSuite:
+    def mem_list(self):
+        return [0] * 256