feat: cross-runtime quantized comparison (pytorch-quantized vs onnx-quantized) #7

Workflow file for this run

	name: ci

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]

	permissions:
	contents: read

	jobs:
	lint:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	cache: pip
	- name: Install lint tools
	run: \|
	python -m pip install --upgrade pip
	python -m pip install ruff==0.4.10 black==24.4.2
	- name: Ruff
	run: ruff check src tests
	- name: Black
	run: black --check src tests

	typecheck:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	cache: pip
	- name: Install
	run: \|
	python -m pip install --upgrade pip
	pip install --index-url https://download.pytorch.org/whl/cpu torch==2.2.2 torchvision==0.17.2
	pip install -e ".[dev]"
	- name: Mypy
	run: mypy src/quant_explorer

	test:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.11", "3.12"]
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: ${{ matrix.python-version }}
	cache: pip
	- name: Install
	run: \|
	python -m pip install --upgrade pip
	pip install --index-url https://download.pytorch.org/whl/cpu torch==2.2.2 torchvision==0.17.2
	pip install -e ".[dev]"
	- name: Unit tests + coverage gate (>= 56%)
	run: pytest tests/unit -v --cov=src/quant_explorer --cov-report=term

	integration-tiny:
	runs-on: ubuntu-latest
	needs: [lint, test]
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	cache: pip
	- name: Install
	run: \|
	python -m pip install --upgrade pip
	pip install --index-url https://download.pytorch.org/whl/cpu torch==2.2.2 torchvision==0.17.2
	pip install -e ".[dev]"
	- name: Cache CIFAR-10 dataset
	uses: actions/cache@v4
	with:
	path: data
	key: cifar10-${{ hashFiles('src/quant_explorer/data.py') }}
	restore-keys: cifar10-
	- name: Tiny pipeline
	env:
	RUN_INTEGRATION: "1"
	run: pytest tests/integration -v

	bench-smoke:
	runs-on: ubuntu-latest
	needs: [lint, test]
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	cache: pip
	- name: Install
	run: \|
	python -m pip install --upgrade pip
	pip install --index-url https://download.pytorch.org/whl/cpu torch==2.2.2 torchvision==0.17.2
	pip install -e ".[dev]"
	- name: Bench fp32_baseline (loads committed weights)
	run: \|
	quant-explorer bench --config fp32_baseline --warmup 2 --iters 20
	- name: Validate result JSON shape
	run: \|
	python - <<'PY'
	import json, pathlib
	p = pathlib.Path("artifacts/results/fp32_baseline.json")
	assert p.exists(), p
	data = json.loads(p.read_text())
	assert "size" in data
	assert "latency" in data and len(data["latency"]) > 0
	assert "memory" in data
	for lat in data["latency"]:
	assert {"batch_size", "p50_ms", "p95_ms", "p99_ms"}.issubset(lat)
	print("ok")
	PY

	qat-smoke:
	runs-on: ubuntu-latest
	needs: [lint, test]
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	cache: pip
	- name: Install
	run: \|
	python -m pip install --upgrade pip
	pip install --index-url https://download.pytorch.org/whl/cpu torch==2.2.2 torchvision==0.17.2
	pip install -e ".[dev]"
	- name: Cache CIFAR-10 dataset
	uses: actions/cache@v4
	with:
	path: data
	key: cifar10-${{ hashFiles('src/quant_explorer/data.py') }}
	restore-keys: cifar10-
	- name: QAT fine-tune (tiny — 256 train images, 1 epoch)
	run: \|
	quant-explorer qat-finetune --epochs 1 --train-subset 256 --batch-size 64
	- name: Bench QAT graph
	run: quant-explorer bench --config qat_int8 --warmup 2 --iters 20
	- name: Validate QAT result JSON
	run: \|
	python - <<'PY'
	import json, pathlib
	p = pathlib.Path("artifacts/results/qat_int8.json")
	assert p.exists(), p
	data = json.loads(p.read_text())
	assert data["size"]["kb"] > 0
	assert all(lat["p50_ms"] > 0 for lat in data["latency"])
	print("ok")
	PY

	cross-runtime-smoke:
	runs-on: ubuntu-latest
	needs: [lint, test]
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	cache: pip
	- name: Install
	run: \|
	python -m pip install --upgrade pip
	pip install --index-url https://download.pytorch.org/whl/cpu torch==2.2.2 torchvision==0.17.2
	pip install -e ".[dev]"
	- name: Cache CIFAR-10 dataset
	uses: actions/cache@v4
	with:
	path: data
	key: cifar10-${{ hashFiles('src/quant_explorer/data.py') }}
	restore-keys: cifar10-
	- name: Cross-runtime comparison (tiny — 2000 test images)
	# 2000 samples keeps the comparison cheap (~15s) while shrinking
	# sampling variance enough that the +/-1pp parity gate is a
	# signal, not noise: on 2k samples, 1pp = 20 disagreements, well
	# above the typical PT-vs-ORT INT8 quantizer drift.
	run: \|
	quant-explorer cross-runtime --accuracy-subset 2000 --calibration-n 128 --warmup 2 --iters 10
	- name: Assert structural parity (top-1 within +/-1pp, all configs)
	run: \|
	python - <<'PY'
	import json, pathlib
	p = pathlib.Path("artifacts/results/cross_runtime.json")
	assert p.exists(), p
	data = json.loads(p.read_text())
	assert data["tolerance_pp"] == 1.0
	rows = data["rows"]
	expected = {
	"fp32_baseline",
	"dynamic_int8",
	"static_int8_per_tensor",
	"static_int8_per_channel",
	}
	assert {r["config"] for r in rows} == expected, [r["config"] for r in rows]
	failures = [
	(r["config"], r["deltas"]["top1_pp"])
	for r in rows
	if not r["deltas"]["within_accuracy_tolerance"]
	]
	assert not failures, f"top-1 parity violated: {failures}"
	for r in rows:
	assert r["pt"]["p50_ms_b1"] > 0
	assert r["onnx"]["p50_ms_b1"] > 0
	assert r["pt"]["size_kb"] > 0
	assert r["onnx"]["size_kb"] > 0
	print("cross-runtime parity ok:", [(r["config"], round(r["deltas"]["top1_pp"], 3)) for r in rows])
	PY
	- name: Validate cross-runtime markdown report
	run: \|
	test -s artifacts/results/cross_runtime.md
	grep -q "Cross-runtime comparison" artifacts/results/cross_runtime.md
	grep -q "SAY-5/onnx-deploy" artifacts/results/cross_runtime.md
	grep -q "SAY-5/export-validator" artifacts/results/cross_runtime.md

	multi-bench-regress:
	runs-on: ubuntu-latest
	needs: [lint, test]
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	cache: pip
	- name: Install
	run: \|
	python -m pip install --upgrade pip
	pip install --index-url https://download.pytorch.org/whl/cpu torch==2.2.2 torchvision==0.17.2
	pip install -e ".[dev]"
	- name: Run multi-model bench (random init, no accuracy)
	run: quant-explorer multi-bench --warmup 1 --iters 5
	- name: Bench-regress structural gate
	run: python scripts/bench_regress_check.py

	build:
	runs-on: ubuntu-latest
	needs: [test]
	steps:
	- uses: actions/checkout@v4
	- name: Build image
	run: docker build -t quant-explorer:ci .
	- name: Smoke run inside container
	run: \|
	# ENTRYPOINT is `quant-explorer`; --help is the safe smoke arg.
	docker run --rm quant-explorer:ci --help \| grep -q "Quant-explorer CLI"
	# Override entrypoint to confirm the package imports cleanly.
	docker run --rm --entrypoint python quant-explorer:ci -c "from quant_explorer.cli import main; print('ok')"

Navigation Menu

feat: cross-runtime quantized comparison (pytorch-quantized vs onnx-quantized) #7

Workflow file

feat: cross-runtime quantized comparison (pytorch-quantized vs onnx-quantized) #7

Uh oh!

Workflow file for this run