Skip to content
Checks

fix(bridge): gate batched-list position_ids on the target model #5964

Sign in to view logs

fix(bridge): gate batched-list position_ids on the target model

fix(bridge): gate batched-list position_ids on the target model #5964

Workflow file for this run

.github/workflows/checks.yml at 6b509c3
name: Checks
on:
workflow_dispatch:
push:
branches:
- main
- dev*
- refactor*
tags:
- "v*.*.*"
paths-ignore:
- ".devcontainer/**"
- ".vscode/**"
- ".git*"
- "*.md"
- ".github/**"
- "!.github/workflows/checks.yml"
pull_request:
branches:
- main
- dev*
- refactor*
paths-ignore:
- ".devcontainer/**"
- ".vscode/**"
- ".git*"
- "*.md"
- ".github/**"
- "!.github/workflows/checks.yml"
# Allow this workflow to be called from other workflows
workflow_call:
inputs:
# Requires at least one input to be valid, but in practice we don't need any
dummy:
type: string
required: false
secrets:
HF_TOKEN:
required: false
permissions:
actions: write
contents: write
# Cancel in-progress PR runs on new push; non-PR events (release, tags) are exempt.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# Retry HF 429s in non-pytest invocations; pytest enables via tests/conftest.py.
env:
TRANSFORMERLENS_HF_RETRY: "1"
# A native dep leaves non-daemon threads alive that hang the process at
# interpreter shutdown after the suite has already passed. Let conftest report
# the lingering threads and hard-exit with the real status once tests finish.
TL_FORCE_EXIT_AFTER_TESTS: "1"
jobs:
compatibility-checks:
name: Compatibility Checks
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version:
- "3.10"
- "3.11"
- "3.12"
steps:
- uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
python-version: ${{ matrix.python-version }}
activate-environment: true
enable-cache: true
- name: Cache Models used with Tests
uses: actions/cache@v4
with:
path: |
~/.cache/huggingface/hub/models--gpt2
~/.cache/huggingface/hub/models--gpt2-xl
~/.cache/huggingface/hub/models--distilgpt2
~/.cache/huggingface/hub/models--EleutherAI--pythia-70m
~/.cache/huggingface/hub/models--EleutherAI--gpt-neo-125M
~/.cache/huggingface/hub/models--google--gemma-2-2b-it
~/.cache/huggingface/hub/models--bigscience--bloom-560m
~/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B
~/.cache/huggingface/hub/models--bert-base-cased
~/.cache/huggingface/hub/models--NeelNanda--Attn_Only*
~/.cache/huggingface/hub/models--roneneldan--TinyStories-1M*
~/.cache/huggingface/hub/models--NeelNanda--SoLU*
~/.cache/huggingface/hub/models--ArthurConmy--redwood_attn_2l
~/.cache/huggingface/hub/models--stas--tiny-random-llama-2
~/.cache/huggingface/hub/models--microsoft--DialoGPT-medium
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-gpt2
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-Olmo2ForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-MptForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-Gemma2ForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-Gemma3ForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-GraniteForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-OlmoForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-OlmoeForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-DeepseekV3ForCausalLM
~/.cache/huggingface/hub/models--katuni4ka--tiny-random-deepseek-v3
~/.cache/huggingface/hub/models--trl-internal-testing--tiny-MistralForCausalLM-0.2
~/.cache/huggingface/hub/models--trl-internal-testing--tiny-BloomForCausalLM
~/.cache/huggingface/hub/models--trl-internal-testing--tiny-Qwen2ForCausalLM-2.5
key: ${{ runner.os }}-huggingface-models-v6
restore-keys: |
${{ runner.os }}-huggingface-models-v5
${{ runner.os }}-huggingface-models-v4
${{ runner.os }}-huggingface-models-v2
${{ runner.os }}-huggingface-models-v1
${{ runner.os }}-huggingface-models
- name: Install dependencies
run: |
uv lock --check
uv sync --extra inspect
- name: Authenticate HuggingFace
if: env.HF_TOKEN != ''
run: uv run python -c "import os; from huggingface_hub import login; login(token=os.environ['HF_TOKEN'])"
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: Unit Test
run: make unit-test
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: Acceptance Test
run: make acceptance-test
- name: Build check
run: uv build
mps-checks:
name: MPS Checks
runs-on: macos-latest
# Only run on PRs merging to main or pushes directly to main
if: >
(github.event_name == 'pull_request' && github.base_ref == 'main') ||
(github.event_name == 'push' && github.ref == 'refs/heads/main')
steps:
- uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
python-version: "3.11"
activate-environment: true
enable-cache: true
- name: MPS Cache Models
uses: actions/cache@v4
with:
path: |
~/.cache/huggingface/hub/models--roneneldan--TinyStories-1M*
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-gpt2
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-Olmo2ForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-MptForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-Gemma2ForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-Gemma3ForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-GraniteForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-OlmoForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-OlmoeForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-DeepseekV3ForCausalLM
~/.cache/huggingface/hub/models--katuni4ka--tiny-random-deepseek-v3
~/.cache/huggingface/hub/models--trl-internal-testing--tiny-MistralForCausalLM-0.2
~/.cache/huggingface/hub/models--trl-internal-testing--tiny-BloomForCausalLM
~/.cache/huggingface/hub/models--trl-internal-testing--tiny-Qwen2ForCausalLM-2.5
key: ${{ runner.os }}-huggingface-models-mps-v2
restore-keys: |
${{ runner.os }}-huggingface-models-mps-v1
- name: Install dependencies
run: |
uv lock --check
uv sync --extra inspect
- name: MPS Availability Check
run: |
uv run python -c "
import torch
print(f'PyTorch: {torch.__version__}')
print(f'MPS available: {torch.backends.mps.is_available()}')
print(f'MPS built: {torch.backends.mps.is_built()}')
assert torch.backends.mps.is_available(), 'MPS not available on this runner!'
"
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: MPS Unit Tests
run: >
uv run pytest tests/unit -v
--ignore=tests/unit/model_bridge/test_optimizer_compatibility.py
--ignore=tests/unit/model_bridge/test_gpt_oss_moe.py
--ignore=tests/unit/model_bridge/test_component_inspection.py
--ignore=tests/unit/model_bridge/test_benchmark_gated_hooks_fire.py
--ignore=tests/unit/model_bridge/test_bridge_generate_kv_cache.py
--ignore=tests/unit/model_bridge/test_bridge_vs_hooked_transformer_patching.py
--ignore=tests/unit/model_bridge/compatibility/
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: MPS Integration Tests
run: >
uv run pytest tests/integration -v -m "not slow"
--ignore=tests/integration/model_bridge/test_optimizer_compatibility.py
--ignore=tests/integration/model_bridge/test_bridge_generation.py
--ignore=tests/integration/model_bridge/test_bridge_integration.py
--ignore=tests/integration/model_bridge/compatibility/
--ignore=tests/integration/test_prepend_bos.py
--ignore=tests/integration/test_generation_compatibility.py
--ignore=tests/integration/test_match_huggingface.py
--ignore=tests/integration/test_fold_layer_integration.py
--ignore=tests/integration/test_centralized_weight_processing.py
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: MPS Smoke Tests
run: uv run pytest tests/mps -v
env:
TRANSFORMERLENS_ALLOW_MPS: "1"
HF_TOKEN: ${{ secrets.HF_TOKEN }}
format-check:
name: Format Check
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
python-version: "3.12"
activate-environment: true
enable-cache: true
- name: Install dependencies
run: |
uv lock --check
uv sync
- name: Check format
run: make check-format
type-check:
name: Type Check
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
python-version: "3.12"
activate-environment: true
enable-cache: true
- name: Install dependencies
run: |
uv lock --check
uv sync
- name: Type check
run: uv run mypy .
docstring-test:
name: Docstring Test
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
python-version: "3.12"
activate-environment: true
enable-cache: true
- name: Install dependencies
run: |
uv lock --check
uv sync
- name: Docstring test
run: make docstring-test
coverage-test:
name: Full Code Coverage Test
runs-on: ubuntu-latest
# Suite runs under pytest-xdist on a 2-vCPU runner (~45 min with swap);
# 75 catches a real hang well below the old serial 90.
timeout-minutes: 75
steps:
- uses: actions/checkout@v4
- name: Add swap space
# The 2-vCPU runner has ~7GB RAM; two xdist workers each holding
# torch + resident models overflow it. Swap absorbs the model-load
# spikes (mirrors the notebook-checks job).
run: |
sudo swapoff /swapfile 2>/dev/null || true
sudo rm -f /swapfile
sudo fallocate -l 8G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
python-version: "3.12"
activate-environment: true
enable-cache: true
- name: Cache Models used with Tests
uses: actions/cache@v4
with:
path: |
~/.cache/huggingface/hub/models--gpt2
~/.cache/huggingface/hub/models--gpt2-xl
~/.cache/huggingface/hub/models--distilgpt2
~/.cache/huggingface/hub/models--EleutherAI--pythia-70m
~/.cache/huggingface/hub/models--EleutherAI--gpt-neo-125M
~/.cache/huggingface/hub/models--google--gemma-2-2b-it
~/.cache/huggingface/hub/models--bigscience--bloom-560m
~/.cache/huggingface/hub/models--Qwen--Qwen2-0.5B
~/.cache/huggingface/hub/models--bert-base-cased
~/.cache/huggingface/hub/models--NeelNanda--Attn_Only*
~/.cache/huggingface/hub/models--roneneldan--TinyStories-1M*
~/.cache/huggingface/hub/models--NeelNanda--SoLU*
~/.cache/huggingface/hub/models--ArthurConmy--redwood_attn_2l
~/.cache/huggingface/hub/models--stas--tiny-random-llama-2
~/.cache/huggingface/hub/models--microsoft--DialoGPT-medium
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-gpt2
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-Olmo2ForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-MptForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-Gemma2ForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-Gemma3ForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-GraniteForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-OlmoForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-OlmoeForCausalLM
~/.cache/huggingface/hub/models--hf-internal-testing--tiny-random-DeepseekV3ForCausalLM
~/.cache/huggingface/hub/models--katuni4ka--tiny-random-deepseek-v3
~/.cache/huggingface/hub/models--trl-internal-testing--tiny-MistralForCausalLM-0.2
~/.cache/huggingface/hub/models--trl-internal-testing--tiny-BloomForCausalLM
~/.cache/huggingface/hub/models--trl-internal-testing--tiny-Qwen2ForCausalLM-2.5
key: ${{ runner.os }}-huggingface-models-v6
restore-keys: |
${{ runner.os }}-huggingface-models-v5
${{ runner.os }}-huggingface-models-v4
${{ runner.os }}-huggingface-models-v2
${{ runner.os }}-huggingface-models-v1
${{ runner.os }}-huggingface-models
- name: Install dependencies
run: |
uv lock --check
uv sync --extra inspect
- name: Authenticate HuggingFace
if: env.HF_TOKEN != ''
run: uv run python -c "import os; from huggingface_hub import login; login(token=os.environ['HF_TOKEN'])"
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: Test Suite with Coverage Report
run: make coverage-report-test
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
# One BLAS thread per worker: N xdist workers each spawning ncores
# OMP threads would oversubscribe the 2-vCPU runner and erase the gain.
OMP_NUM_THREADS: "1"
MKL_NUM_THREADS: "1"
# Worker count knob (this runner is 2-vCPU, so -n auto = 2). If swap
# still can't hold peak memory, drop to "-n 1" or grow the swapfile.
XDIST_ARGS: "-n auto --dist loadscope"
- name: Build check
run: uv build
- name: Upload Coverage Report Artifact
uses: actions/upload-artifact@v7
with:
name: test-coverage
path: htmlcov
benchmark-checks:
name: Benchmark Tests
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
python-version: "3.12"
activate-environment: true
enable-cache: true
- name: Cache Models used with Tests
uses: actions/cache@v4
with:
path: |
~/.cache/huggingface/hub/models--gpt2
# Distinct key: this job needs only gpt2, and sharing the full-suite
# key let whichever job saved first pin the cache contents for all
# three (a one-path save then serves the suites that need the rest).
key: ${{ runner.os }}-huggingface-models-benchmark-v1
- name: Install dependencies
run: |
uv lock --check
uv sync
- name: Authenticate HuggingFace
if: env.HF_TOKEN != ''
run: uv run python -c "import os; from huggingface_hub import login; login(token=os.environ['HF_TOKEN'])"
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: Benchmark Test
run: make benchmark-test
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
notebook-checks:
name: Notebook Checks
runs-on: ubuntu-latest
timeout-minutes: 30
env:
HAS_HF_TOKEN: ${{ secrets.HF_TOKEN != '' }}
strategy:
fail-fast: false
matrix:
notebook:
- "Attribution_Patching_Demo"
- "Activation_Patching_in_TL_Demo"
- "ARENA_Content"
- "BERT"
- "Bridge_Evals_Demo"
- "Exploratory_Analysis_Demo"
# - "Grokking_Demo"
- "Head_Detector_Demo"
# - "Interactive_Neuroscope"
- "Main_Demo"
# - "No_Position_Experiment"
- "Othello_GPT"
- "Patchscopes_Generation_Demo"
- "Santa_Coder"
# - "stable_lm"
- "T5"
requires_hf_token: [false]
include:
- notebook: "LLaMA"
requires_hf_token: true
- notebook: "LLaMA_GPU_Quantized"
requires_hf_token: true
steps:
- uses: actions/checkout@v4
- name: Add swap space
run: |
sudo swapoff /swapfile 2>/dev/null || true
sudo rm -f /swapfile
sudo fallocate -l 8G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
python-version: "3.11"
activate-environment: true
enable-cache: true
- name: Re-use HuggingFace models cache
uses: actions/cache/restore@v4
with:
path: ~/.cache/huggingface/hub
key: ${{ runner.os }}-huggingface-models
- name: Install dependencies
run: |
uv lock --check
uv sync --group quantization --extra evals
- name: Install pandoc
uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: pandoc
version: 1.0
- name: Authenticate HuggingFace
if: env.HF_TOKEN != ''
run: uv run python -c "import os; from huggingface_hub import login; login(token=os.environ['HF_TOKEN'])"
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: Check Notebook Output Consistency
if: ${{ !matrix.requires_hf_token || env.HAS_HF_TOKEN == 'true' }}
run: pytest --nbval-sanitize-with demos/doc_sanitize.cfg demos/${{ matrix.notebook }}.ipynb
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: Skip (HF_TOKEN not available)
if: ${{ matrix.requires_hf_token && env.HAS_HF_TOKEN != 'true' }}
run: echo "Skipping ${{ matrix.notebook }} — requires HF_TOKEN for gated model access"
build-docs:
# When running on a PR, this just checks we can build the docs without errors
# When running on merge to main, it builds the docs and then another job deploys them
name: 'Build Docs'
runs-on: ubuntu-latest
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev') || contains(github.head_ref, 'docs')
needs: coverage-test
steps:
- uses: actions/checkout@v4
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
python-version: "3.11"
activate-environment: true
enable-cache: true
- name: Install pandoc
uses: awalsh128/cache-apt-pkgs-action@latest
with:
packages: pandoc
version: 1.0
- name: Install dependencies
run: |
uv lock --check
uv sync
- name: Download Test Coverage Artifact
uses: actions/download-artifact@v8
with:
name: test-coverage
path: docs/source/_static/coverage
- name: Authenticate HuggingFace
if: env.HF_TOKEN != ''
run: uv run python -c "import os; from huggingface_hub import login; login(token=os.environ['HF_TOKEN'])"
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: Build Docs
run: make build-docs
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
- name: Upload Docs Artifact
uses: actions/upload-artifact@v7
with:
name: documentation
path: docs/build
deploy-docs:
name: Deploy Docs
runs-on: ubuntu-latest
# Only run if merging a PR into main
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
needs: build-docs
steps:
- uses: actions/checkout@v4
- name: Download Docs Artifact
uses: actions/download-artifact@v8
with:
name: documentation
path: docs/build
- name: Upload to GitHub Pages
uses: JamesIves/github-pages-deploy-action@v4
with:
folder: docs/build
clean-exclude: |
*.*.*/