pytest-asan.yml

# Copyright (c) ONNX Project Contributors
#
# SPDX-License-Identifier: Apache-2.0

name: Pytest with AddressSanitizer

on:
  push:
    branches:
      - main
  pull_request:
  workflow_dispatch:

permissions:
  contents: read

jobs:
  pytest-asan:
    name: Pytest (ASan)
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          submodules: recursive
          persist-credentials: false

      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: "3.12"

      - name: Install build dependencies
        run: python -m pip install --upgrade pip protobuf setuptools wheel "cmake>=3.22"

      - name: Build and install onnxoptimizer with AddressSanitizer
        env:
          # Build the native extension (and the bundled onnx / protobuf) with
          # ASan. The flags are single-quoted so setup.py's shlex.split keeps
          # each multi-token value as one -D... argument. -g / frame pointers
          # give readable stack traces; the linker flags make libasan a
          # DT_NEEDED of the shared objects.
          ASAN_FLAGS: "-fsanitize=address -fno-omit-frame-pointer -g"
        run: |
          export CXXFLAGS=-fsanitize=address LDFLAGS=-fsanitize=address
          python -m pip install --no-build-isolation -v .
          # The test step runs `pytest onnxoptimizer/test/` from the repo root, so
          # `import onnxoptimizer` binds to the in-tree source package instead of the
          # installed one -- and the source tree has no compiled extension. Build it
          # in place as well (reusing the cmake build pip just produced) so the
          # onnx_opt_cpp2py_export module sits next to that source; without this the
          # tests fail at collection with "ModuleNotFoundError: No module named
          # 'onnxoptimizer.onnx_opt_cpp2py_export'".
          python setup.py build_ext --inplace

      - name: Install test dependencies
        # onnxruntime is intentionally omitted: it is only used by the tests for
        # optional numerical comparison (guarded by `has_ort`), so skipping it
        # keeps ASan focused on onnxoptimizer's own C++ while still running every
        # optimize() call and model check.
        run: python -m pip install pytest pytest-xdist numpy

      - name: Run pytest under AddressSanitizer
        env:
          # CPython itself is not built with ASan, so the runtime has to be
          # preloaded; otherwise dlopen of the instrumented extension fails
          # with "ASan runtime does not come first in initial library list".
          # detect_leaks=0 suppresses leaks from the uninstrumented interpreter
          # and third-party libraries so the job only fails on genuine memory
          # errors (heap-buffer-overflow, use-after-free, ...) in onnxoptimizer.
          ASAN_OPTIONS: "detect_leaks=0:halt_on_error=1:symbolize=1"
        run: |
          # libasan must come first in the preload list (ASan requirement), but
          # libstdc++ must be preloaded alongside it: the `onnx` Python wheel is
          # an uninstrumented C++ extension that throws C++ exceptions (e.g. from
          # checker.check_model / shape_inference, which the tests exercise
          # constantly). Without libstdc++ loaded at ASan init, the __cxa_throw
          # interceptor can't resolve the real symbol and aborts the process with
          # "CHECK failed: ... real___cxa_throw != 0" on the first raised
          # exception. Preloading libstdc++ lets the interceptor find __cxa_throw
          # so exceptions unwind normally and only genuine memory errors fail CI.
          export LD_PRELOAD="$(gcc -print-file-name=libasan.so):$(gcc -print-file-name=libstdc++.so.6)"
          # Limit worker count to keep ASan's larger memory footprint in check.
          python -m pytest onnxoptimizer/test/ -n 2 -v