Cargo.toml

# There is deliberately no `default-members` here, and adding one is a bigger
# change than it looks. The workspace root is itself the `mlxcel` package, so a
# bare `cargo <cmd>` run here resolves to `-p mlxcel`; `default-members` would
# re-point every such invocation in the repository at once, release.yml's
# `cargo build --release --target aarch64-apple-darwin --locked` included, which
# would start compiling the default-off `mlxcel-xla` into every release build.
#
# The consequence is that a command must say `--workspace` to see all five
# members, and until #1007 the quality gate did not: `make verify-test` and
# `make verify-clippy` covered the root package only, leaving 1754 tests across
# the others (1354 of them mlxcel-core's) and the whole of their test-target
# lint surface ungated. Both targets now pass `--workspace` explicitly. If you
# add a sixth member, they pick it up with no change here.
#
# `mlxcel-mlx-pin` is a leaf with no production role: it hosts the unit tests
# for the MLX-pin logic in `mlxcel-core/build_support/mlx_pin.rs` so that logic
# can be tested without compiling mlxcel-core (and therefore MLX C++). Nothing
# depends on it; see its manifest for why it is a path include rather than a
# build dependency.
[workspace]
members = [
    ".",
    "src/lib/mlxcel-core",
    "src/lib/mlxcel-mlx-pin",
    "src/lib/mlxcel-surgery",
    "src/lib/mlxcel-xla",
]
resolver = "2"

[package]
name = "mlxcel"
version = "0.7.0-beta.1"
edition = "2024"
description = "High-performance LLM/VLM/VLA inference on Apple Silicon and CUDA GPUs"
repository = "https://github.com/lablup/mlxcel"
license = "Apache-2.0"
keywords = ["mlx", "llm", "vlm", "inference"]
categories = ["science", "command-line-utilities"]
# No-op unless the `xla-iree` feature is on, where it emits the IREE runtime link
# recipe for the binary (a dependency's link-args do not propagate). See build.rs.
build = "build.rs"

[[bin]]
name = "mlxcel"
path = "src/main.rs"

[[bin]]
name = "mlxcel-server"
path = "src/bin/mlx_server.rs"

# Perf benchmark harness for the speculative drafter pairings.
# Captures no-drafter baseline tok/s and scaffolds
# the speculative numerator rows that follow-up will fill in. See
# `src/bin/speculative_bench.rs` for the module docs and `docs/model_tests.md`
# (`Speculative drafters`) for the deferral matrix.
[[bin]]
name = "speculative_bench"
path = "src/bin/speculative_bench.rs"

# Same-process decode benchmark harness used by `scripts/bench_decode.sh`.
# It loads a model once, runs the warmup generation, resets model/cache state,
# then runs the measured pass in the same process. This mirrors the Python
# `stream_generate` timing more closely than two separate `mlxcel generate`
# invocations, especially for prefill.
[[bin]]
name = "mlxcel-bench-decode"
path = "src/bin/bench_decode.rs"

[features]
default = ["surgery"]
metal = ["mlxcel-core/metal"]
accelerate = ["mlxcel-core/accelerate"]
cuda = ["mlxcel-core/cuda"]
test-utils = []
# Seam scaffold for an optional non-MLX forward-execution backend (issue #338).
#
# Default OFF. Shipping binaries (Apple Silicon, CUDA) compile no backend code
# beyond the MLX path: the `crate::backend::experimental` module and the
# `Backend::Experimental` enum variant are `cfg`-gated behind this feature, and
# `select_backend()` folds to the single `Backend::Mlx` variant (a zero-sized
# enum) with no runtime dispatch on the load or forward path. Enabling this
# feature compiles the scaffold plug-in point where a future non-MLX engine
# (the motivating target is FuriosaAI TCP / RNGD, which cannot route through
# MLX) implements `ComputeBackend`. No kernels ship with this feature today; it
# only lands the boundary.
experimental-backend = []
# OpenXLA / StableHLO compiler-family backend (issue #449, ADR 0004 Track B).
#
# Default OFF. Pulls in the `mlxcel-xla` crate and compiles the `crate::backend::xla`
# module, the `Backend::Xla` / `Session::Xla` enum variants, and the
# `MLXCEL_BACKEND=xla` selection arm. Apple-Silicon and CUDA shipping builds do
# not enable it, so they compile none of it and the seam folds to MLX as before.
xla-backend = ["dep:mlxcel-xla"]
# Real OpenXLA execution: `xla-backend` plus the `mlxcel-xla/iree` feature, which
# compiles the C shim against the prebuilt IREE runtime and drives the bundled
# StableHLO prefill / decode_step graphs. Needs IREE_DIST at build time (the
# extracted iree dist), so it is a local / opt-in build, not a CI default.
xla-iree = ["xla-backend", "mlxcel-xla/iree"]
# Explicit test-only forwarding for Gemma3n intermediate oracle validation.
# Its pinned reference is the production CUDA runtime, so enabling diagnostics
# also enables the root MLX CUDA backend. This is not part of `xla-iree`; normal
# production bundles remain unchanged.
xla-diagnostics = ["cuda", "xla-iree", "mlxcel-xla/diagnostics"]
# CPU-capable bounded MLX/IREE operator-oracle harness. CUDA-specific production
# probes may add `cuda`, but the shared report/comparison layer does not require it.
xla-micro-oracle = ["xla-iree", "mlxcel-xla/micro-oracle"]
# CPU-only companion for bounded host-reference checks on machines where the
# CUDA driver is unavailable. This deliberately does not replace the qualified
# `xla-diagnostics` production-target gate.
xla-diagnostics-cpu = ["xla-iree", "mlxcel-xla/diagnostics"]
# Axis A "weight-load surgery" framework.
#
# On by default so production `mlxcel` and `mlxcel-server` binaries expose
# the `--surgery <config.yaml>` flag without rebuilds. Bit-exactness with
# the pre-surgery baseline is preserved without conditional features: when
# the flag is not supplied, the active-pipeline slot in `crate::surgery`
# stays `None`, the consolidated loaders call `transform.apply` zero times,
# and the load path matches `transform = None` byte-for-byte (verified by
# `crate::models::sanitize_tests::load_text_weights_with_none_transform_matches_legacy_path`).
#
# Build with `--no-default-features` to compile without the surgery crate
# entirely (e.g. CI parity tests against pre-A1 behaviour or constrained
# embedded targets).
surgery = ["dep:mlxcel-surgery"]

[dependencies]
# MLX - mlxcel-core (direct C++ bindings via cxx)
mlxcel-core = { path = "src/lib/mlxcel-core", default-features = false }

# Axis A weight-load surgery — opt-in via the `surgery` feature above.
mlxcel-surgery = { path = "src/lib/mlxcel-surgery", optional = true }

# OpenXLA / StableHLO compiler-family backend — opt-in via the `xla-backend`
# feature below. Default OFF; never enters Apple-Silicon or CUDA shipping builds.
mlxcel-xla = { path = "src/lib/mlxcel-xla", optional = true }

# Serialization
serde = { version = "1.0", features = ["derive"] }
serde_json = { version = "1.0", features = ["preserve_order"] }
serde_yaml = "0.9"

# Tensors & Models
safetensors = "0.7.0"
tokenizers = "0.22.2"
sentencepiece = "0.13"
hf-hub = "0.5.0"

# Zero-copy buffers
bytes = "1.9"
memmap2 = "0.9"

# LZ4 compression (pure Rust, used for tensor transfer)
lz4_flex = "0.11"

# Error Handling
anyhow = "1.0.100"
thiserror = "2.0.11"

# Cryptographic hashing (for vision feature cache keys)
sha2 = "0.10"

# Fast non-cryptographic-grade hashing for prompt prefix cache keys.
# BLAKE3 is used only for in-process cache-key digests; no security boundary
# depends on it.
blake3 = "1.8"

# CLI
clap = { version = "4.6.1", features = ["derive", "env"] }

# Interactive chat REPL line editing (epic #92, issue #96). `rustyline`
# provides the readline-style line editor (history, cursor movement,
# Ctrl-C / Ctrl-D handling) for `mlxcel`'s interactive chat loop. Only the
# offline CLI REPL uses it; the server is unaffected. Default features pull
# in the terminal backend so arrow-key history works out of the box.
rustyline = "14"

# Image processing (for VLM)
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"] }

# Compressed-audio container decoding for the transcription routes (#1446).
# llama-server b10621's mtmd audio front-end takes wav, mp3 and flac; mlxcel
# decodes RIFF/WAVE itself (src/audio/preprocessing_wav.rs) and delegates the
# two compressed containers here. Pure Rust, no C dependency, MPL-2.0 (allowed
# by deny.toml). Only the two codecs b10621 accepts are enabled, so the
# dependency does not drag in aac/alac/isomp4/ogg decoders nobody reaches.
symphonia = { version = "0.5", default-features = false, features = ["mp3", "flac"] }

# Base64 (for VLM image data URIs in server API)
base64 = "0.22"

# Utils
# `dirs` resolves the platform home directory for the downloader's HF-cache
# read-reuse default (`~/.cache/huggingface/hub`) when neither `HF_HUB_CACHE`
# nor `HF_HOME` is set (issue #93). Already present transitively via
# mlxcel-core; promoted to a direct dependency so `src/downloader/store.rs`
# can call `dirs::home_dir()` without leaning on a transitive re-export. The
# mlxcel store root itself comes from `mlxcel_core::cache_root()`, which uses
# the same crate.
dirs = "6"
glob = "0.3.2"
path-clean = "1.0.1"
minijinja = { version = "2.20", features = ["loop_controls", "fuel", "json", "preserve_order"] }
# Used by `src/downloader/mod.rs::file_url` to percent-encode `repo_id` /
# `revision` / `filename` path segments before composing the HF download URL
# (L1). Already transitively present via reqwest -> url; promoting
# to a direct dep so the downloader can call into it without depending on
# reqwest's transitive re-export.
percent-encoding = "2"

# Structured outputs / constrained decoding.
#
# `llguidance` is the same library used by mlx-vlm upstream PR #1047 for the
# JSON-schema response_format MVP. The Rust crate exposes `Matcher` /
# `ParserFactory` directly so we don't need to round-trip through Python.
# `toktrie_hf_tokenizers` adapts our existing HuggingFace `tokenizers::Tokenizer`
# into the byte-level `TokEnv` that llguidance expects.
#
# `default-features = false, features = ["lark", "referencing"]` drops the
# optional `rayon` thread pool and the `ahash` non-deterministic hasher; we
# keep `lark` (string-form grammar parsing — used by `from_tagged_str`) and
# `referencing` (JSON-schema $ref resolution required for nested schemas).
llguidance = { version = "1.7", default-features = false, features = ["lark", "referencing"] }
toktrie_hf_tokenizers = "1.7"

# HTTP Server
tokio = { version = "1.52", features = ["full"] }
axum = { version = "0.7", features = ["json", "macros", "multipart"] }
tokio-stream = "0.1"
tokio-util = "0.7"
futures = "0.3"
uuid = { version = "1.23", features = ["v4"] }
chrono = "0.4"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
# The `cors` feature is deliberately absent: #1432 replaced the tower-http
# CORS layer with the b10621-shaped middleware in `src/server/cors.rs`, which
# has to set `Access-Control-Allow-Origin` on every response and answer
# `OPTIONS` before authentication and before routing.
tower-http = { version = "0.5", features = ["trace"] }
tower = { version = "0.4", features = ["util"] }
hyper = { version = "1.10", features = ["server"] }
hyper-util = { version = "0.1", features = ["tokio", "server-auto"] }
# HTTPS listener for `--ssl-cert-file` / `--ssl-key-file` (#1432). `ring` is
# selected explicitly and `aws-lc-rs` (the tokio-rustls default) is left off:
# reqwest already resolves rustls onto the ring provider in this workspace, and
# pulling a second provider in would both add a C toolchain dependency to the
# build and make rustls's process-default provider ambiguous at runtime.
tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "ring", "tls12"] }
# PEM parsing for the `--ssl-cert-file` / `--ssl-key-file` material (#1432).
# `rustls-pemfile` is the older spelling of the same code and is unmaintained
# (RUSTSEC-2025-0134); its own advisory points here. Already in the graph
# through rustls.
rustls-pki-types = { version = "1", features = ["std"] }
# `http_body::Body` is implemented directly for the read-budget body wrapper
# that gates the `--timeout` socket read deadline (#1432). Already in the graph
# through hyper and axum; promoted so the trait can be named.
http-body = "1"
# SO_REUSEPORT for `--reuse-port` (#1432); the `all` feature is what exposes
# `Socket::set_reuse_port`. Already in the dependency graph through tokio.
socket2 = { version = "0.6", features = ["all"] }
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] }
indicatif = "0.18"
async-stream = "0.3"
fancy-regex = "0.17.0"
toml = "0.8"

[target.'cfg(unix)'.dependencies]
# Used for the RDMA-aware transport backend capability probe (io_uring on
# Linux, kqueue / writev batched sends on macOS). Unix-only: this does not
# affect Windows builds of the library today.
libc = "0.2"

[profile.release]
strip = true              # Strip all symbols for smaller binary
lto = true                # Full (fat) LTO for best size and performance
codegen-units = 1         # Single codegen unit for maximum optimization
opt-level = 3             # Maximum speed optimization (inference is compute-bound)
# Unwind (not abort) so the deliberate `catch_unwind` request-isolation backstop
# works in release as it does in debug: the audio worker `run_guarded`
# (src/server/audio_worker.rs) turns a synthesis panic into a per-request error
# instead of aborting the server. That audio worker is the only contained
# boundary. Every core inference worker thread re-imposes fail-fast explicitly
# with a targeted `catch_unwind` + `process::abort()` (see `run_core_thread_or_abort`
# in src/worker_failfast.rs and ADR 0003): the batched and legacy server workers
# (src/server/model_worker.rs) and the remote pipeline stage service thread
# (src/distributed/pipeline/remote_service.rs), so a broken core invariant still
# crashes cleanly for a supervised restart rather than silently unwinding.
panic = "unwind"

# Fast profile for local/agent edit-test iteration (issue #809).
#
# [profile.release] above is tuned for shipping binaries: fat LTO across all
# ~439 locked crates plus a single codegen unit (`codegen-units = 1`) for the
# ~390k-line main crate maximizes runtime performance and minimizes binary
# size, at the cost of a multi-minute link on every rebuild, measured at
# 4 to 6 minutes per incremental rebuild, and a typical issue cycle pays 5 to
# 8 such rebuilds. See docs/installation.md ("Fast iteration builds") for the
# full measurement writeup.
#
# `test-fast` keeps `opt-level = 3` (MLX-heavy tests need optimized numerics
# for tolerable runtimes) but relaxes everything that exists purely to shrink
# or speed up the *shipped* binary:
#   - `codegen-units = 16` restores parallel codegen instead of serializing
#     the whole crate onto one core.
#   - `lto = false` disables cross-crate LTO for test binaries. The full
#     workspace gate has enough large integration targets that ThinLTO can
#     fail in the macOS linker with missing internal Rust drop-glue symbols,
#     while the shipped release profile keeps its fat-LTO tuning.
#   - `incremental = true` lets rustc reuse per-function codegen work between
#     edits instead of recompiling the whole crate query graph from scratch.
#   - `strip = false` skips a symbol-stripping pass nobody needs for a local
#     test binary.
#
# DO NOT use this profile for anything that ships or gets measured:
# performance benchmarks, release binaries, or numbers quoted in a PR as
# representative throughput/latency. Use `[profile.release]` (via
# `make release*`) for those.
#
# As of #1000 this profile also backs `make verify-test`, so it is what the
# CI-faithful gate and the nightly workflow run tests under. That is a
# deliberate trade: `opt-level = 3` is what makes the optimised-codegen
# argument for testing hold, and it is kept, while cross-crate LTO and the single
# codegen unit that were consuming the nightly's entire budget in linking are
# not. As of #1406 it also disables cross-crate ThinLTO for test binaries
# because large macOS integration-test link graphs can otherwise fail before
# any test runs. It does not change what ships; release.yml still builds the
# shipped binaries under `[profile.release]`.
[profile.test-fast]
inherits = "release"
lto = false            # no cross-crate LTO for test binaries; release stays fat-LTO
codegen-units = 16     # parallel codegen instead of one serialized unit
strip = false          # skip the strip pass; irrelevant for a local test binary
incremental = true     # reuse per-function codegen work between edits
opt-level = 3          # keep: MLX-heavy tests need optimized numerics for tolerable runtimes

[lints.clippy]
upper_case_acronyms = "allow"
too_many_arguments = "allow"
type_complexity = "allow"
large_enum_variant = "allow"
should_implement_trait = "allow"

[dev-dependencies]
ctor = "1.0.13"
tempfile = "3"
# `#[tokio::test(start_paused = true)]` for the socket-timeout unit tests
# (#1432); `full` does not include the test clock.
tokio = { version = "1.52", features = ["full", "test-util"] }
# Self-signed certificates for the `--ssl-cert-file` / `--ssl-key-file` tests
# (#1432). Generating the keypair in-test keeps a private key out of the
# repository; `ring` matches the provider the server itself pins.
rcgen = { version = "0.13", default-features = false, features = ["crypto", "pem", "ring"] }
# `BodyExt::collect` for the read-budget body tests (#1432). Already in the
# graph through axum and hyper-util.
http-body-util = "0.1"
criterion = "0.5"

[[bench]]
name = "audio_fft"
harness = false

[[test]]
name = "distributed_integration"
required-features = ["test-utils"]

[[test]]
name = "pipeline_e2e"
required-features = ["test-utils"]

[[test]]
name = "paged_handoff_parity"
required-features = ["test-utils"]

[[test]]
name = "xla_prepared_prefill"
required-features = ["xla-iree"]

[[test]]
name = "molmo2_xla_vision_parity"
required-features = ["xla-iree"]

# Reference-equivalence + throughput harness for the OpenXLA/IREE continuous
# batching engine (#449 M3 Stage 2b). Needs real IREE execution, so it builds
# only under `xla-iree` (which carries the runtime link recipe); default/CI
# builds skip it.
[[example]]
name = "xla_batch_bench"
required-features = ["xla-iree"]

# Token-exactness gate vs an external greedy oracle (#449 M3 Stage 2d, Stage B).
# Drives the single-sequence OpenXLA engine over a prompt and asserts its greedy
# token stream equals a reference captured from HF transformers, proving the
# emitted graph computes the architecture correctly (the Qwen2 QKV-bias + plain
# RoPE gate). Needs real IREE execution, so it builds only under `xla-iree`.
[[example]]
name = "xla_oracle_check"
required-features = ["xla-iree"]

# Dump the OpenXLA engine's greedy argmax trajectory as an oracle JSON at the
# ambient MLXCEL_XLA_PRECISION, so the token-exactness gate above can use the XLA
# f32 path itself as the reference (issue #575, Metal validation without the HF
# fp32 Python oracle). Needs real IREE execution, so it builds only under `xla-iree`.
[[example]]
name = "xla_traj_dump"
required-features = ["xla-iree"]

# Independent LLaVA stage capture against the pinned HF oracle (#862).
# Host vision/projector diagnostics require the qualified CUDA-backed MLX path,
# while text prefill/decode run on the selected IREE CPU or CUDA device.
[[example]]
name = "xla_llava_reference_check"
required-features = ["xla-diagnostics"]

# Ordered first-divergence gate for the native IREE LLaVA vision/projector.
# It consumes the independently generated #862 oracle and is intentionally
# excluded from default builds because it links the diagnostic IREE runtime.
[[example]]
name = "xla_vision_reference_check"
required-features = ["xla-diagnostics"]

[[example]]
name = "xla_aux_smoke"
required-features = ["xla-iree"]

# Bounded canonical-decomposition vs emitted IREE operator probes. The root
# example owns the static IREE runtime link recipe, while the reusable harness
# remains in mlxcel-xla behind the CPU-capable micro-oracle feature.
[[example]]
name = "xla_numeric_probe"
required-features = ["xla-micro-oracle"]

# Pinned #874 MLX-vs-IREE Phi4MM Conformer and speech/vision projection parity.
[[example]]
name = "xla_phi4_audio_check"
required-features = ["xla-iree"]