fix: extraction-engine audit (#1223) (#1225) #815
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI E2E | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - "crates/**" | |
| - "packages/**" | |
| - "e2e/**" | |
| - "fixtures/**" | |
| - "alef.toml" | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "Taskfile.yml" | |
| - "pnpm-workspace.yaml" | |
| - ".github/workflows/ci-e2e.yaml" | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - "crates/**" | |
| - "packages/**" | |
| - "e2e/**" | |
| - "fixtures/**" | |
| - "alef.toml" | |
| - "Cargo.toml" | |
| - "Cargo.lock" | |
| - "Taskfile.yml" | |
| - "pnpm-workspace.yaml" | |
| - ".github/workflows/ci-e2e.yaml" | |
| workflow_dispatch: | |
| concurrency: | |
| group: ci-e2e-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_INCREMENTAL: 0 | |
| CARGO_PROFILE_DEV_DEBUG: 0 | |
| RUST_BACKTRACE: full | |
| # Swift crashes at the FFI boundary print a symbolicated backtrace instead of a | |
| # bare SIGABRT, so a failing e2e run yields actionable data (see the swift | |
| # Linux-ARM64 Option<handle> decode crash). | |
| SWIFT_BACKTRACE: "enable=yes,sanitize=yes,cache=yes" | |
| RUST_MIN_STACK: 16777216 | |
| ORT_VERSION: "1.24.2" | |
| MACOSX_DEPLOYMENT_TARGET: "14.0" | |
| BUILD_PROFILE: "ci" | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" | |
| permissions: | |
| contents: read | |
| jobs: | |
| build-ffi: | |
| name: Build FFI (${{ matrix.target }}) | |
| if: github.repository == 'xberg-io/xberg' && github.actor != 'dependabot[bot]' | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: ${{ matrix.os == 'windows-latest' && 120 || 60 }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| - os: ubuntu-24.04-arm | |
| target: aarch64-unknown-linux-gnu | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| submodules: recursive | |
| # Build FFI is the slowest job in ci-e2e (full cargo release with --features all). | |
| # Hash every file that actually affects the FFI ABI or the embedded core lib, then | |
| # restore from cache when unchanged. The hash also gates the build steps below. | |
| - name: Compute FFI source hash | |
| id: ffi-hash | |
| shell: bash | |
| run: | | |
| hash=$(find \ | |
| Cargo.toml Cargo.lock rust-toolchain.toml \ | |
| crates/xberg \ | |
| crates/xberg-ffi \ | |
| crates/xberg-cli \ | |
| crates/xberg-tesseract \ | |
| crates/xberg-paddle-ocr \ | |
| -type f \ | |
| \( -name '*.rs' -o -name 'Cargo.toml' -o -name 'Cargo.lock' \ | |
| -o -name 'build.rs' -o -name '*.h' -o -name '*.c' \ | |
| -o -name '*.pc' -o -name 'rust-toolchain.toml' \) \ | |
| 2>/dev/null | LC_ALL=C sort | xargs sha256sum 2>/dev/null | sha256sum | cut -d' ' -f1) | |
| echo "value=$hash" >> "$GITHUB_OUTPUT" | |
| echo "FFI source hash: $hash" | |
| - name: Restore FFI artifacts from cache | |
| id: ffi-cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| target/release/libxberg_ffi.* | |
| target/release/xberg_ffi.* | |
| target/release/xberg | |
| target/release/xberg.exe | |
| crates/xberg-ffi/include/xberg.h | |
| crates/xberg-ffi/xberg-ffi.pc | |
| crates/xberg-ffi/cmake/ | |
| key: ffi-${{ matrix.target }}-${{ steps.ffi-hash.outputs.value }}-libheif-v1.23.0-cache-v2 | |
| - name: Setup Rust | |
| if: steps.ffi-cache.outputs.cache-hit != 'true' | |
| uses: xberg-io/actions/setup-rust@v1 | |
| with: | |
| cache-key-prefix: build-ffi-${{ matrix.target }} | |
| target: ${{ matrix.target }} | |
| - name: Install system dependencies | |
| if: steps.ffi-cache.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/install-system-deps | |
| - name: Build FFI library | |
| if: steps.ffi-cache.outputs.cache-hit != 'true' | |
| uses: xberg-io/actions/build-rust-ffi@v1 | |
| with: | |
| crate-name: xberg-ffi | |
| - name: Build CLI | |
| if: steps.ffi-cache.outputs.cache-hit != 'true' | |
| uses: xberg-io/actions/build-rust-cli@v1 | |
| with: | |
| package-name: xberg-cli | |
| binary-name: xberg | |
| extra-cargo-args: --features all | |
| - name: Upload FFI artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ffi-${{ matrix.target }} | |
| path: | | |
| target/release/libxberg_ffi.* | |
| target/release/xberg_ffi.* | |
| crates/xberg-ffi/include/xberg.h | |
| crates/xberg-ffi/xberg-ffi.pc | |
| crates/xberg-ffi/cmake/ | |
| target/release/xberg | |
| target/release/xberg.exe | |
| retention-days: 7 | |
| if-no-files-found: error | |
| e2e-tests: | |
| name: E2E (${{ matrix.lang }}) | |
| if: github.repository == 'xberg-io/xberg' && github.actor != 'dependabot[bot]' | |
| needs: [build-ffi] | |
| runs-on: ${{ matrix.runs-on }} | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - lang: python | |
| python-version: "3.13" | |
| runs-on: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| - lang: node | |
| node-version: "24" | |
| runs-on: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| - lang: go | |
| go-version: "1.26" | |
| runs-on: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| - lang: ruby | |
| ruby-version: "3.4" | |
| runs-on: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| - lang: java | |
| java-version: "25" | |
| runs-on: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| - lang: csharp | |
| dotnet-version: "10.0.x" | |
| runs-on: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| - lang: php | |
| php-version: "8.4" | |
| runs-on: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| - lang: elixir | |
| elixir-version: "1.19" | |
| otp-version: "28" | |
| runs-on: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| - lang: wasm | |
| node-version: "24" | |
| runs-on: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| - lang: rust | |
| runs-on: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| - lang: dart | |
| dart-version: "3.11" | |
| runs-on: ubuntu-24.04-arm | |
| target: aarch64-unknown-linux-gnu | |
| - lang: kotlin-android | |
| java-version: "25" | |
| runs-on: ubuntu-24.04-arm | |
| target: aarch64-unknown-linux-gnu | |
| - lang: swift | |
| swift-version: "6.0" | |
| runs-on: ubuntu-24.04-arm | |
| target: aarch64-unknown-linux-gnu | |
| - lang: zig | |
| zig-version: "0.16.0" | |
| runs-on: ubuntu-24.04-arm | |
| target: aarch64-unknown-linux-gnu | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| submodules: recursive | |
| - name: Setup Rust | |
| uses: xberg-io/actions/setup-rust@v1 | |
| with: | |
| cache-key-prefix: e2e-${{ matrix.lang }} | |
| use-sccache: "true" | |
| - name: Download FFI artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: ffi-${{ matrix.target }} | |
| path: ffi-artifacts | |
| - name: Stage FFI artifacts | |
| shell: bash | |
| run: | | |
| mkdir -p target/release crates/xberg-ffi/include crates/xberg-ffi/cmake | |
| if [ -d ffi-artifacts/target/release ]; then | |
| cp -r ffi-artifacts/target/release/. target/release/ | |
| fi | |
| if [ -d ffi-artifacts/crates/xberg-ffi/include ]; then | |
| cp -r ffi-artifacts/crates/xberg-ffi/include/. crates/xberg-ffi/include/ | |
| fi | |
| if [ -d ffi-artifacts/crates/xberg-ffi/cmake ]; then | |
| cp -r ffi-artifacts/crates/xberg-ffi/cmake/. crates/xberg-ffi/cmake/ | |
| fi | |
| if [ -f ffi-artifacts/crates/xberg-ffi/xberg-ffi.pc ]; then | |
| cp ffi-artifacts/crates/xberg-ffi/xberg-ffi.pc crates/xberg-ffi/ | |
| fi | |
| chmod +x target/release/libxberg_ffi.so 2>/dev/null || true | |
| ls -la target/release/ | |
| if [ -f target/release/libxberg_ffi.so ]; then | |
| sudo cp target/release/libxberg_ffi.so /usr/local/lib/ | |
| sudo ldconfig | |
| fi | |
| - name: Install system dependencies | |
| uses: ./.github/actions/install-system-deps | |
| - name: Verify libheif installation (Linux) | |
| if: runner.os == 'Linux' | |
| shell: bash | |
| env: | |
| PKG_CONFIG_PATH: /usr/local/lib/pkgconfig:${PKG_CONFIG_PATH:-} | |
| LD_LIBRARY_PATH: /usr/local/lib:${LD_LIBRARY_PATH:-} | |
| run: | | |
| echo "Checking libheif installation..." | |
| if [ -f /usr/local/lib/pkgconfig/libheif.pc ]; then | |
| version=$(pkg-config --modversion libheif 2>/dev/null || echo "unknown") | |
| echo "✓ Found libheif at /usr/local/lib (version: $version)" | |
| elif pkg-config --exists libheif; then | |
| version=$(pkg-config --modversion libheif) | |
| location=$(pkg-config --variable=libdir libheif) | |
| echo "⚠ libheif found at $location (version: $version), not /usr/local/lib" | |
| if [ "$(printf '%s\n' "$version" "1.21" | sort -V | head -n1)" = "1.21" ] && [ "$version" != "1.21" ]; then | |
| echo "::notice::System libheif version $version >= 1.21, should be compatible" | |
| else | |
| echo "::warning::System libheif version $version < 1.21, may cause ABI mismatches" | |
| fi | |
| else | |
| echo "::error::libheif not found on system" | |
| exit 1 | |
| fi | |
| echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" | |
| echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH" | |
| - name: Setup ONNX Runtime | |
| uses: ./.github/actions/setup-onnx-runtime | |
| with: | |
| ort-version: ${{ env.ORT_VERSION }} | |
| strategy: system | |
| - name: Stage ORT libraries alongside FFI | |
| shell: bash | |
| run: | | |
| if [ -n "${ORT_LIB_LOCATION:-}" ] && [ -d "$ORT_LIB_LOCATION" ]; then | |
| mkdir -p target/release | |
| cp -f "$ORT_LIB_LOCATION"/libonnxruntime.so* target/release/ || true | |
| echo "Staged ORT libraries to target/release:" | |
| ls -la target/release/libonnxruntime.so* 2>/dev/null || echo "No ORT libraries found (may be bundled)" | |
| fi | |
| - name: Stage FFI libraries for C# e2e | |
| if: matrix.lang == 'csharp' | |
| shell: bash | |
| run: | | |
| mkdir -p e2e/csharp/bin/Debug/net10.0 | |
| cp -f target/release/libxberg_ffi.so e2e/csharp/bin/Debug/net10.0/ 2>/dev/null || true | |
| if [ -f target/release/libonnxruntime.so.1 ]; then | |
| cp -f target/release/libonnxruntime.so.1 e2e/csharp/bin/Debug/net10.0/ || true | |
| fi | |
| ls -la e2e/csharp/bin/Debug/net10.0/lib*.so 2>/dev/null || echo "No native libraries found for C#" | |
| - name: Setup Tesseract cache | |
| uses: ./.github/actions/setup-tesseract-cache | |
| with: | |
| label: e2e-${{ matrix.lang }} | |
| - name: Install WASI SDK | |
| if: matrix.lang == 'wasm' | |
| uses: xberg-io/actions/install-wasi-sdk@v1 | |
| - name: Setup Python | |
| if: matrix.python-version | |
| uses: xberg-io/actions/setup-python-env@v1 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache-prefix: e2e-py-${{ matrix.python-version }} | |
| - name: Setup Node | |
| if: matrix.node-version | |
| uses: xberg-io/actions/setup-node-workspace@v1 | |
| - name: Setup Go | |
| if: matrix.go-version | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: ${{ matrix.go-version }} | |
| - name: Setup Ruby | |
| if: matrix.ruby-version | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: ${{ matrix.ruby-version }} | |
| bundler-cache: false | |
| working-directory: e2e/ruby | |
| - name: Bundle install | |
| if: matrix.ruby-version | |
| working-directory: e2e/ruby | |
| run: | | |
| bundle config set frozen true | |
| bundle install --jobs 4 | |
| - name: Setup Java | |
| if: matrix.java-version | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: temurin | |
| java-version: ${{ matrix.java-version }} | |
| - name: Setup Android SDK | |
| if: matrix.lang == 'kotlin-android' | |
| uses: android-actions/setup-android@v3 | |
| with: | |
| # setup-android@v3 dropped the `api-level` / `build-tools-version` | |
| # inputs (they were ignored with a workflow warning) and the legacy | |
| # `sdkmanager tools` install path now fails with exit code 1 because | |
| # Google removed the "tools" package. Use the modern `packages` | |
| # input to declare exactly what we need. | |
| packages: "platforms;android-35 build-tools;35.0.0" | |
| accept-android-sdk-licenses: true | |
| - name: Setup Gradle | |
| if: matrix.lang == 'kotlin-android' | |
| uses: xberg-io/actions/setup-gradle@v1 | |
| with: | |
| gradle-version: "9.6.0" | |
| - name: Setup .NET | |
| if: matrix.dotnet-version | |
| uses: actions/setup-dotnet@v5 | |
| with: | |
| dotnet-version: ${{ matrix.dotnet-version }} | |
| - name: Setup PHP | |
| if: matrix.php-version | |
| uses: xberg-io/actions/setup-php@v1 | |
| with: | |
| php-version: ${{ matrix.php-version }} | |
| tools: composer | |
| - name: Setup Elixir | |
| if: matrix.elixir-version | |
| uses: xberg-io/actions/setup-elixir@v1 | |
| with: | |
| elixir-version: ${{ matrix.elixir-version }} | |
| otp-version: ${{ matrix.otp-version }} | |
| - name: Setup Dart | |
| if: matrix.dart-version | |
| uses: dart-lang/setup-dart@v1 | |
| with: | |
| sdk: ${{ matrix.dart-version }} | |
| - name: Setup Swift | |
| if: matrix.swift-version | |
| uses: xberg-io/actions/setup-swift@v1 | |
| with: | |
| swift-version: ${{ matrix.swift-version }} | |
| - name: Setup Zig | |
| if: matrix.zig-version | |
| uses: xberg-io/actions/setup-zig@v1 | |
| with: | |
| version: ${{ matrix.zig-version }} | |
| - name: Setup library paths for FFI bindings | |
| if: | | |
| matrix.lang == 'go' || matrix.lang == 'java' || | |
| matrix.lang == 'csharp' || matrix.lang == 'elixir' || | |
| matrix.lang == 'kotlin-android' || | |
| matrix.lang == 'swift' || matrix.lang == 'zig' || | |
| matrix.lang == 'rust' || matrix.lang == 'python' | |
| shell: bash | |
| run: | | |
| export PKG_CONFIG_PATH="${PWD}/crates/xberg-ffi:${PKG_CONFIG_PATH}" | |
| # Ensure /usr/local/lib is on LD_LIBRARY_PATH for source-built system libs | |
| # (e.g., libheif 1.23.0 from install-system-deps, not apt's older version) | |
| export LD_LIBRARY_PATH="/usr/local/lib:${PWD}/target/release:${ORT_LIB_LOCATION}:${LD_LIBRARY_PATH}" | |
| echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" >> "$GITHUB_ENV" | |
| echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> "$GITHUB_ENV" | |
| - name: Install Task | |
| uses: xberg-io/actions/install-task@v1 | |
| - name: Install alef CLI | |
| uses: xberg-io/actions/install-alef@v1 | |
| - name: Pre-build mock-server for Go e2e | |
| if: matrix.lang == 'go' | |
| shell: bash | |
| run: | | |
| echo "Pre-building mock-server for Go e2e tests..." | |
| cargo build --release --manifest-path e2e/rust/Cargo.toml --bin mock-server | |
| mkdir -p /tmp/xberg-e2e | |
| cp e2e/rust/target/release/mock-server /tmp/xberg-e2e/mock-server | |
| echo "XBERG_E2E_MOCK_SERVER_BIN=/tmp/xberg-e2e/mock-server" >> "$GITHUB_ENV" | |
| - name: Compile Ruby native extension | |
| if: matrix.lang == 'ruby' | |
| working-directory: packages/ruby | |
| run: bundle install && bundle exec rake compile | |
| - name: Run e2e tests | |
| run: task ${{ matrix.lang }}:e2e | |
| shell: bash | |
| env: | |
| PKG_CONFIG_PATH: ${{ env.PKG_CONFIG_PATH }} | |
| LD_LIBRARY_PATH: ${{ env.LD_LIBRARY_PATH }} | |
| DYLD_LIBRARY_PATH: ${{ env.DYLD_LIBRARY_PATH || '' }} | |
| TESSDATA_PREFIX: "/usr/share/tesseract-ocr/5/tessdata" | |
| # Diagnostics for a failed e2e run — logs, generated sources, and any crash | |
| # reports. Combined with RUST_BACKTRACE=full / SWIFT_BACKTRACE above, a failure | |
| # yields actionable data instead of a bare exit code. | |
| - name: Upload e2e diagnostics on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: e2e-diagnostics-${{ matrix.lang }} | |
| path: | | |
| e2e/**/*.log | |
| e2e/**/*.trace | |
| e2e/**/TestResults/** | |
| e2e/**/test-results/** | |
| e2e/**/build/reports/** | |
| e2e/**/hs_err_pid*.log | |
| if-no-files-found: ignore | |
| retention-days: 7 |