diff --git a/.anvil.lock b/.anvil.lock index a127283c..baaf8b9b 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.6.0" -catalog_checksum = "sha256:411e8b3d247721c856c6a8a69f7b388711159325472ab687eff9c40bdd9727b0" +catalog_checksum = "sha256:06529602514854ab5676975141a7542dc31a2577f7615ddd8ea516ed74d41bd5" [[file]] path = ".anvil/container/Dockerfile.dockerignore" @@ -121,19 +121,19 @@ checksum = "sha256:f4822a8f9cb282bc426790355ba83b63e778db9327e986ba3c86ae6205cfd [[file]] path = "justfiles/anvil/checks/miri-race-coverage.just" -checksum = "sha256:f8435d359f4495a3a1b3693069b0bab371bf94c3698389d9749acf95704a3c45" +checksum = "sha256:eb6d41a49ccd72ec533aeb336103835383b81fc07c9ed38329eb09bcdc807444" [[file]] path = "justfiles/anvil/checks/miri-strict-provenance.just" -checksum = "sha256:2e3fafdb060f10635a0dc34b447db4f06d7dd1549e585440611a5018a1e4ef5c" +checksum = "sha256:cc5fa94c7eb0f6cef28a885c3e72ea4277b5b42e3cbe8e2d750fa91d9a579109" [[file]] path = "justfiles/anvil/checks/miri-tree-borrows.just" -checksum = "sha256:c82f586402f2cb05397a40eec6fe95c7590df2cc52f0dd9da6879de08517561c" +checksum = "sha256:1df669672fb1a507ae087750a3d7872a96990d5b2410c4843217152939963375" [[file]] path = "justfiles/anvil/checks/miri.just" -checksum = "sha256:9e9d0cbfef1e1e1586c2af4e9c387719c2f017ea1203d326baacc3ae25df80e2" +checksum = "sha256:91b67bc68bcc34a82233917112b297935fdedccf46ffd668884636a79cc4579d" [[file]] path = "justfiles/anvil/checks/msrv-test.just" @@ -169,7 +169,7 @@ checksum = "sha256:6efd7378a2cd0f5d86519bd32fd86f2055a60191187dd77a8842b374b8eb7 [[file]] path = "justfiles/anvil/container.just" -checksum = "sha256:3363992c6c006c649eae3732b2a7b04c2a4925eff369179b128d43380e2800d4" +checksum = "sha256:a772d84b20c796fb37d283781a956916097783d1a8e9a132e4e933fc8fc23865" [[file]] path = "justfiles/anvil/groups/pr-fast.just" diff --git a/crates/cargo-anvil/README.md b/crates/cargo-anvil/README.md index d3eb1fb2..ca171ecf 100644 --- a/crates/cargo-anvil/README.md +++ b/crates/cargo-anvil/README.md @@ -380,8 +380,10 @@ metadata conventions — see its documentation. #### Undefined-behavior checking (`miri`) -The PR-tier `miri` check runs `cargo miri test --all-features --tests` -(libtest, not nextest — process-per-test is roughly twice as slow under miri). +The PR-tier `miri` check compiles `cargo miri test --all-features --tests` +once, then runs the resulting libtest artifacts concurrently through +`cargo-miri runner`. Set `ANVIL_MIRI_JOBS` to a positive integer to override +the default of one worker per logical processor. Opt a test out of miri when it touches the filesystem, spawns subprocesses, or otherwise can’t run under the interpreter: @@ -400,6 +402,18 @@ under tree-borrows): #[cfg_attr(miri_race_coverage, ignore = "nondeterministic across seeds")] ``` +A package whose own test targets should not run under Miri can opt out while +remaining available as a dependency: + +```toml +[package.metadata.anvil.miri] +exclude = true +``` + +Reserve this package-wide switch for constraints that apply to every test +target. Prefer per-test ignores with reasons for narrower or temporary +suppressions. + #### Concurrency model checking (`loom`) The `loom` check runs only the test targets that opt in, detected @@ -491,7 +505,7 @@ And `docs/verification.md` for the continuous-validation strategy. This crate was developed as part of The Oxidizer Project. Browse this crate's source code. - [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjNhdIQbFhzZ8rzWNNYbuRaDSGWynFgbH4PMdoT7GNcbVwNPtPjAhvFhYvRhcoQbLvVGTNtetQUbnp9vX0Ew7_gbkZEyxfXZXyMbltL72AXa-o1hZIGDa2NhcmdvLWFudmlsZTAuNi4wa2NhcmdvX2Fudmls + [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjNhdIQbFhzZ8rzWNNYbuRaDSGWynFgbH4PMdoT7GNcbVwNPtPjAhvFhYvRhcoQbKr_fs3dc0BAbULxaTH4TvIUbQX7XkOZRWOQbkl8wl0Pm6CNhZIGDa2NhcmdvLWFudmlsZTAuNi4wa2NhcmdvX2Fudmls [__link0]: https://crates.io/crates/cargo-delta [__link1]: https://docs.rs/cargo-anvil/0.6.0/cargo_anvil/?search=artifacts::container [__link10]: https://docs.rs/cargo-anvil/0.6.0/cargo_anvil/?search=artifacts diff --git a/crates/cargo-anvil/docs/design/checks.md b/crates/cargo-anvil/docs/design/checks.md index 7ccb2499..ee84141a 100644 --- a/crates/cargo-anvil/docs/design/checks.md +++ b/crates/cargo-anvil/docs/design/checks.md @@ -236,7 +236,7 @@ available. | Check | Invocation | Source | |-----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------| -| `miri` | `cargo + miri test --all-features --tests` over the impact-affected packages. Uses libtest (one process per test binary), **not** `cargo miri nextest run`: under miri, nextest's process-per-test model pays miri's expensive std-initialization re-interpretation for *every* test and roughly doubles wall-time on a large suite (the dominant cost on the PR critical path). `--tests` runs lib/bin unit tests and integration tests (the same target set nextest ran) while excluding doctests, which miri can't run; it is used in preference to `--lib --tests` because `--lib` errors with "no library targets found" on a bin-only affected package under impact scoping. Slow tests opt out per-test with `#[cfg_attr(miri, ignore)]` -- anvil doesn't pass exotic `MIRIFLAGS`; the per-test opt-out is the canonical mechanism. libtest exits 0 when a binary's tests are all skipped, so no `--no-tests=pass` workaround is needed. The recipe reads its scope from the `target/anvil/impact/` cache via `_anvil-impact-include`; because it depends on `anvil-impact`, a clean direct or PR invocation is impact-scoped (unaffected packages are skipped). It runs the full workspace only when scoping is off — the scheduled/full tiers set `ANVIL_IMPACT=off`, and a dirty local tree widens for safety. | oxidizer, oxidizer-github | +| `miri` | Compiles `cargo + miri test --all-features --tests --no-run` once over the impact-affected packages, parses Cargo's JSON artifact stream, then runs each compiled Miri test executable concurrently through `cargo-miri runner`. Compiling the selected scope together preserves Cargo feature unification: shared dependencies receive the union of features requested by all selected packages, matching a normal workspace-scoped Cargo invocation. The default is one artifact worker per logical processor, clamped to the executable count; `ANVIL_MIRI_JOBS` accepts a positive integer override. Output is isolated per executable and labelled with its package and test target, then replayed deterministically; every executable completes before failures are aggregated. Only compiler artifacts with `profile.test = true` run, excluding build scripts and ordinary executables. `--tests` selects lib/bin unit tests and integration tests while excluding doctests, benches, and examples. Slow or unsupported individual tests opt out with `#[cfg_attr(miri, ignore)]`. A package can omit all its own Miri targets through `[package.metadata.anvil.miri] exclude = true`; it may still compile as a dependency of a selected consumer. Clean direct and PR invocations are impact-scoped; scheduled/full tiers and dirty local trees run the full workspace. | oxidizer, oxidizer-github | | `careful` | `cargo + careful test --all-features --locked` over the impact-affected packages. cargo-careful uses a debug-instrumented std in a stable cache path. Because Cargo fingerprints the sysroot path rather than its contents, the recipe records the actual `rustc -vV` and SHA-256 of the resolved `cargo-careful` executable in `target/anvil/careful-sysroot.id`; either changing triggers `cargo clean`. The executable hash is used because cargo-careful rejects version-only invocations. This remains correct when validation accepts a newer installed cargo-careful. | oxidizer-github | | `loom` | For each `[[test]]` target that declares `required-features = ["loom"]`, `cargo test -p --release --all-features --locked --test -- --test-threads=1` with `RUSTFLAGS="--cfg loom"`. [`loom`](https://crates.io/crates/loom) is a permutation-based concurrency model checker that explores thread interleavings. Anvil does not impose a global exploration bound; each model remains responsible for tractable exhaustive exploration. Targets are detected **structurally** from `cargo metadata` (a test target whose `kind` contains `test` and whose `required-features` contains `loom`) -- not via a filename/cfg/comment heuristic -- and only those targets run, so loom never touches a crate's ordinary tests. The `loom` feature selects the target (`required-features`); `--cfg loom` activates loom (source swaps std↔loom atomics on `#[cfg(loom)]`, and `[target.'cfg(loom)'.dependencies] loom` links only under the cfg) -- both are required. Scoped per-package with `-p` (never `--workspace`) so the global cfg never leaks into deps reachable only through other members. **Fail-loud**: a crate that declares loom support (a `loom` feature or a `cfg(loom)` dependency) but exposes no such test target errors out rather than silently no-opping. When no crate ships a loom target the recipe skips (exit 0). | oxidizer-github | | `bolero` | Uses the catalog nightly and release profile consistently to discover targets one package at a time, then runs each affected libfuzzer target for 60 seconds on Linux. Explicitly selecting `release` avoids cargo-bolero's implicit, adopter-defined `fuzz` profile and matches target execution. Adopters that disable `bolero`'s default features must enable its `std` feature for libfuzzer support. Per-package discovery is required because `cargo-bolero list` accepts only one `--package`; local whole-workspace runs enumerate workspace members before discovery. A successful empty discovery is a no-op; metadata, discovery, or parsing failure fails the check. Non-Linux hosts skip because cargo-bolero's native dependencies are unsupported there, while harnesses still run as ordinary tests. | oxidizer-github | @@ -288,11 +288,11 @@ commit can't surface anything new.) | Check | Invocation | Source | |--------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------| | `miri` | Same recipe as the `pr-runtime-analysis` member, but the `scheduled-runtime-analysis` group forces `ANVIL_IMPACT=off` (emit-time `__IMPACT_MODE__`) so the run is full-workspace. PR-tier miri is impact-scoped (so a PR touching crate A never exercises crate B under miri); the scheduled re-run ensures every crate gets miri coverage on `main` at least daily, catching UB introduced by an inter-crate change whose PR happened to scope it out. | oxidizer, oxidizer-github | -| `miri-tree-borrows` | `MIRIFLAGS='-Zmiri-tree-borrows' RUSTFLAGS='--cfg miri_tree_borrows' cargo + miri test --all-features --tests`. Tree-borrows tracks per-byte aliasing provenance and can exceed the 16 GB Linux runner; tests known to OOM under tree-borrows are quarantined per-test in source via `#[cfg_attr(miri_tree_borrows, ignore = "")]` so the suppression lives next to the test rather than in a sidecar file. The recipe declares the cfg name via `--check-cfg=cfg(miri_tree_borrows)` so non-miri builds don't warn. | oxidizer-github (rewritten as cfg-based) | -| `miri-strict-provenance` | `MIRIFLAGS='-Zmiri-strict-provenance' RUSTFLAGS='--cfg miri_strict_provenance' cargo + miri test --all-features --tests`. Surfaces integer-to-pointer casts that don't satisfy strict provenance; complementary to tree-borrows. Per-test opt-outs use `#[cfg_attr(miri_strict_provenance, ignore = "")]`. | oxidizer-github | -| `miri-race-coverage` | `MIRIFLAGS="-Zmiri-many-seeds=.." RUSTFLAGS='--cfg miri_race_coverage' cargo + miri test --all-features --tests`. The `..` window rotates daily based on day-of-month (day N -> seeds `2N-1..2N+1`, exclusive upper bound -> 2 seeds/day, ~62 seeds/month). Rotating amortizes the seed space across the schedule rather than retesting the same seeds every night; race conditions surface as inter-seed nondeterminism rather than per-seed crashes, so coverage matters more than depth-per-seed. Per-test opt-outs use `#[cfg_attr(miri_race_coverage, ignore = "")]`. | oxidizer-github | +| `miri-tree-borrows` | Uses the shared compile-once artifact runner with `MIRIFLAGS='-Zmiri-tree-borrows'` and `RUSTFLAGS='--cfg miri_tree_borrows'`. Tree-borrows tracks per-byte aliasing provenance and can exceed runner memory; tests known to OOM are quarantined per-test via `#[cfg_attr(miri_tree_borrows, ignore = "")]`. | oxidizer-github (rewritten as cfg-based) | +| `miri-strict-provenance` | Uses the shared compile-once artifact runner with `MIRIFLAGS='-Zmiri-strict-provenance'` and `RUSTFLAGS='--cfg miri_strict_provenance'`. It surfaces integer-to-pointer casts that do not satisfy strict provenance; per-test opt-outs use `#[cfg_attr(miri_strict_provenance, ignore = "")]`. | oxidizer-github | +| `miri-race-coverage` | Uses the shared compile-once artifact runner with `MIRIFLAGS="-Zmiri-many-seeds=.."` and `RUSTFLAGS='--cfg miri_race_coverage'`. The `..` window rotates daily based on day-of-month (day N -> seeds `2N-1..2N+1`, exclusive upper bound -> 2 seeds/day, about 62 seeds/month). Rotation keeps each scheduled run bounded while exploring different nondeterministic schedules over successive days instead of repeating one fixed seed set. Per-test opt-outs use `#[cfg_attr(miri_race_coverage, ignore = "")]`. | oxidizer-github | -These profiles each cost hours per leg (oxidizer caps `miri-race-coverage` at 12 h), which is why they live in scheduled rather than PR. They share `miri`'s setup and run sequentially within the `scheduled-runtime-analysis` group. Each uses a profile-specific `MIRIFLAGS` (the actual miri mode, e.g. `-Zmiri-tree-borrows`) plus a profile-specific `--cfg miri_` in `RUSTFLAGS`; the distinct cfg is what lets a test opt out of just one profile via `#[cfg_attr(miri_, ignore = "…")]` without affecting the others. The OS matrix matches `pr-runtime-analysis` (4 legs on GitHub, 2 on ADO) so any OS already considered "worth running miri on" gets the harder profiles too -- the single-tier-per-group rule forbids running tree-borrows on a strict subset of the OSes where stacked-borrows runs, which would silently hide tree-borrows-only UB on the dropped legs. +These profiles each cost hours per leg, which is why they live in scheduled rather than PR. They share `miri`'s setup and run sequentially within the `scheduled-runtime-analysis` group; only independent test artifacts inside one profile run concurrently. Each uses profile-specific `MIRIFLAGS` plus a profile-specific `--cfg miri_` in `RUSTFLAGS`, letting a test opt out of one profile without affecting the others. The OS matrix matches `pr-runtime-analysis` (4 legs on GitHub, 2 on ADO) so any OS considered worth running Miri on gets the harder profiles too. Per-test opt-outs live in source via `#[cfg_attr(miri_, ignore = "")]`. Each miri-profile recipe sets the matching `--cfg` in `RUSTFLAGS`; the cfg names are also declared in the workspace lints region (`unexpected_cfgs` + `check-cfg`) so non-miri builds don't warn. This keeps the suppression next to the test (and behind code review) rather than in a sidecar file the build system has to parse out-of-band. diff --git a/crates/cargo-anvil/docs/design/containers.md b/crates/cargo-anvil/docs/design/containers.md index b4753c5b..be374d65 100644 --- a/crates/cargo-anvil/docs/design/containers.md +++ b/crates/cargo-anvil/docs/design/containers.md @@ -373,12 +373,13 @@ alone. The driver therefore follows each nested target a plan names, until nothi tier reads as needing nothing and `anvil-aprz` runs unauthenticated inside an image that has no `gh` of its own. It also forwards the recipe contract's own inputs when they are set — `PR_TITLE`, `BASE_REF`, `GITHUB_BASE_REF`, -`SYSTEM_PULLREQUEST_TARGETBRANCH` and `ANVIL_IMPACT` — because a check that reads one natively must read -the same value in a container. `anvil-pr-title` is the sharp case: with `PR_TITLE` unset it exits 0 with a skip notice, -so dropping it at the boundary would let a title a native run rejects pass in a container while the tier still reported -green. `ANVIL_IMPACT` is the other: a CI group job exports `consume`, and a container that did not inherit it would -recompute scoping from a diff instead of trusting the artifact the group downloaded. They are forwarded by name and -only when set, so an unset variable stays unset rather than arriving empty. +`SYSTEM_PULLREQUEST_TARGETBRANCH`, `ANVIL_IMPACT` and `ANVIL_MIRI_JOBS` — because a check that reads one +natively must read the same value in a container. `anvil-pr-title` is the sharp case: with `PR_TITLE` unset it exits 0 +with a skip notice, so dropping it at the boundary would let a title a native run rejects pass in a container while the +tier still reported green. `ANVIL_IMPACT` controls whether a CI group trusts its downloaded impact artifact. +`ANVIL_MIRI_JOBS` caps the number of memory-heavy Miri artifact workers; dropping it could turn a deliberately bounded +container run into one worker per logical processor. They are forwarded by name and only when set, so an unset variable +stays unset rather than arriving empty. A resolved token is set on the driver process, passed by name, and unset after the run, so it never reaches a host command line. Inside the container it is readable by everything the run executes, including build scripts and proc diff --git a/crates/cargo-anvil/docs/design/local.md b/crates/cargo-anvil/docs/design/local.md index f2aecb66..74dfb2fc 100644 --- a/crates/cargo-anvil/docs/design/local.md +++ b/crates/cargo-anvil/docs/design/local.md @@ -671,7 +671,31 @@ baseline — so an environment without the base ref must either provide it, run `ANVIL_IMPACT=off`, or (in CI) download the cache and set `ANVIL_IMPACT=consume`. -## 5. Daily driver +## 5. Miri execution customization + +Miri compiles the selected package scope once, then executes each compiled Miri +test executable concurrently. `ANVIL_MIRI_JOBS` overrides the default of one +artifact worker per logical processor and must be a positive integer. The worker +count is always clamped to the number of discovered executables. Use the override +when a runner needs a lower memory footprint; containerized runs forward the +setting unchanged. + +Packages whose own test targets are inherently unsuitable or unproductive under +Miri can declare: + +```toml +[package.metadata.anvil.miri] +exclude = true +``` + +This Boolean disables every Miri test target owned by the package, in both +impact-scoped and full-workspace runs, although the package can still compile as +a dependency of another selected package. Because the metadata has no place to +record a reason, reserve it for package-wide constraints. Prefer per-test +`cfg_attr(miri, ignore = "")` or the profile-specific cfgs when a narrower +or temporary suppression can keep its rationale beside the affected test. + +## 6. Daily driver ```text $ just anvil @@ -685,7 +709,7 @@ anvil OK (`anvil-pr`, `anvil-scheduled`, `anvil-full`) are first-class -- locally reproducible with exactly the same arguments cloud workflows uses, because cloud workflows invokes the same `just` recipes. -## 6. No-tooling fallback +## 7. No-tooling fallback A user with only `cargo` (no `just`, no `cargo-anvil`) can still run the basics: @@ -700,7 +724,7 @@ The same commands appear as the body of the corresponding `just` recipes under covers core hygiene only — coverage, miri, mutants, etc. still require their respective tools. -## 7. Customization at the recipe level +## 8. Customization at the recipe level Per the four customization tiers in [README.md §7](./README.md#7-customization): diff --git a/crates/cargo-anvil/src/anvil/artifacts/container.rs b/crates/cargo-anvil/src/anvil/artifacts/container.rs index 8e01f276..a7a320df 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/container.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/container.rs @@ -831,6 +831,7 @@ mod tests { "GITHUB_BASE_REF", "SYSTEM_PULLREQUEST_TARGETBRANCH", "ANVIL_IMPACT", + "ANVIL_MIRI_JOBS", ] { assert!(RECIPE.contains(name), "{name} must be forwarded"); } @@ -839,8 +840,8 @@ mod tests { // container that did not inherit it would recompute from a diff instead // of trusting the artifact the group downloaded. assert!( - RECIPE.contains("'ANVIL_IMPACT')) {"), - "ANVIL_IMPACT must be in the forwarded set, not merely mentioned" + RECIPE.contains("'ANVIL_IMPACT', 'ANVIL_MIRI_JOBS')) {"), + "ANVIL_IMPACT and ANVIL_MIRI_JOBS must be in the forwarded set, not merely mentioned" ); // Nothing reads these; forwarding them only implied a contract that // does not exist. See justfile.rs, which asserts they stay removed. diff --git a/crates/cargo-anvil/src/anvil/artifacts/justfile.rs b/crates/cargo-anvil/src/anvil/artifacts/justfile.rs index 7af41063..4fed54a5 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/justfile.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/justfile.rs @@ -399,6 +399,43 @@ mod tests { } } + #[test] + fn miri_profiles_delegate_to_the_parallel_artifact_runner() { + let miri = CHECK_FILES + .iter() + .find_map(|(path, body)| path.ends_with("/miri.just").then_some(*body)) + .expect("miri.just is registered in CHECK_FILES"); + for needle in [ + "_anvil-miri-test *package_args:", + "miri test --all-features --tests --no-run --message-format=json-render-diagnostics", + "$message.profile.test -ne $true", + "ForEach-Object -Parallel", + "-ThrottleLimit $jobs", + "rustc $toolchain --print sysroot", + "anvil miri: ANVIL_MIRI_JOBS must be a positive integer", + "Sort-Object PackageId, TargetKind, TargetName, Path", + "##[group]Miri artifact", + "::group::Miri artifact", + ] { + assert!(miri.contains(needle), "miri runner is missing '{needle}'"); + } + assert!( + miri.contains("$env:MIRI_BE_RUSTC = 'host'"), + "standalone cargo-miri runner calls need the same ambient rustc mode recorded at build time" + ); + + for check in ["miri", "miri-tree-borrows", "miri-strict-provenance", "miri-race-coverage"] { + let body = CHECK_FILES + .iter() + .find_map(|(path, body)| path.ends_with(&format!("/{check}.just")).then_some(*body)) + .unwrap_or_else(|| panic!("{check}.just is registered in CHECK_FILES")); + assert!( + body.contains("& \"{{ just_executable() }}\" _anvil-miri-test @pkg"), + "{check}.just must delegate to the shared Miri artifact runner" + ); + } + } + #[test] fn impact_recipe_is_defined_and_reuses_shared_helpers() { // The single impact building block: snapshot + compute + resolve. diff --git a/crates/cargo-anvil/src/lib.rs b/crates/cargo-anvil/src/lib.rs index 31fd7881..3f287052 100644 --- a/crates/cargo-anvil/src/lib.rs +++ b/crates/cargo-anvil/src/lib.rs @@ -381,8 +381,10 @@ //! //! ### Undefined-behavior checking (`miri`) //! -//! The PR-tier `miri` check runs `cargo miri test --all-features --tests` -//! (libtest, not nextest — process-per-test is roughly twice as slow under miri). +//! The PR-tier `miri` check compiles `cargo miri test --all-features --tests` +//! once, then runs the resulting libtest artifacts concurrently through +//! `cargo-miri runner`. Set `ANVIL_MIRI_JOBS` to a positive integer to override +//! the default of one worker per logical processor. //! Opt a test out of miri when it touches the filesystem, spawns //! subprocesses, or otherwise can't run under the interpreter: //! @@ -401,6 +403,18 @@ //! #[cfg_attr(miri_race_coverage, ignore = "nondeterministic across seeds")] //! ``` //! +//! A package whose own test targets should not run under Miri can opt out while +//! remaining available as a dependency: +//! +//! ```toml +//! [package.metadata.anvil.miri] +//! exclude = true +//! ``` +//! +//! Reserve this package-wide switch for constraints that apply to every test +//! target. Prefer per-test ignores with reasons for narrower or temporary +//! suppressions. +//! //! ### Concurrency model checking (`loom`) //! //! The `loom` check runs only the test targets that opt in, detected diff --git a/crates/cargo-anvil/templates/justfiles/anvil/checks/miri-race-coverage.just b/crates/cargo-anvil/templates/justfiles/anvil/checks/miri-race-coverage.just index 4851799b..de4a2716 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/checks/miri-race-coverage.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/checks/miri-race-coverage.just @@ -26,7 +26,7 @@ anvil-miri-race-coverage: anvil-miri-race-coverage-validate-prereqs anvil-impact $env:MIRIFLAGS = "-Zmiri-many-seeds=$low..$high $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_race_coverage $($env:RUSTFLAGS)".Trim() Write-Host "anvil-miri-race-coverage: seed window $low..$high (day $day)" - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Install prerequisites for the `anvil-miri-race-coverage` recipe. diff --git a/crates/cargo-anvil/templates/justfiles/anvil/checks/miri-strict-provenance.just b/crates/cargo-anvil/templates/justfiles/anvil/checks/miri-strict-provenance.just index 60f36c27..65de202b 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/checks/miri-strict-provenance.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/checks/miri-strict-provenance.just @@ -15,7 +15,7 @@ anvil-miri-strict-provenance: anvil-miri-strict-provenance-validate-prereqs anvi $pkg = @(if ($include) { -split $include } else { '--workspace' }) $env:MIRIFLAGS = "-Zmiri-strict-provenance $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_strict_provenance $($env:RUSTFLAGS)".Trim() - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Install prerequisites for the `anvil-miri-strict-provenance` recipe. diff --git a/crates/cargo-anvil/templates/justfiles/anvil/checks/miri-tree-borrows.just b/crates/cargo-anvil/templates/justfiles/anvil/checks/miri-tree-borrows.just index c1786354..eccdcd8f 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/checks/miri-tree-borrows.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/checks/miri-tree-borrows.just @@ -29,7 +29,7 @@ anvil-miri-tree-borrows: anvil-miri-tree-borrows-validate-prereqs anvil-impact $pkg = @(if ($include) { -split $include } else { '--workspace' }) $env:MIRIFLAGS = "-Zmiri-tree-borrows $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_tree_borrows $($env:RUSTFLAGS)".Trim() - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # The three nightly-miri profiles share the same toolchain prereqs diff --git a/crates/cargo-anvil/templates/justfiles/anvil/checks/miri.just b/crates/cargo-anvil/templates/justfiles/anvil/checks/miri.just index 630e2c79..10f5ca6b 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/checks/miri.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/checks/miri.just @@ -12,6 +12,249 @@ # scheduled/full tiers (which force ANVIL_IMPACT=off) or a dirty local tree # (which widens for safety). +# Compile all selected packages together, then run their Miri test artifacts in +# parallel. The single Cargo invocation preserves workspace feature unification, +# while artifact-level parallelism uses host logical processors that +# `cargo miri test` otherwise leaves idle. +# +# Packages can opt their own test targets out through: +# +# [package.metadata.anvil.miri] +# exclude = true +# +# They can still compile as dependencies of selected consumers. Workspace runs +# translate the metadata to `--exclude`; impact-scoped runs remove matching +# `--package name@version` pairs before compilation. +# +# ANVIL_MIRI_JOBS overrides the default of one artifact worker per logical processor. + +# Compile selected packages once and run their Miri test artifacts concurrently. +[script("pwsh", "-NoProfile")] +_anvil-miri-test *package_args: + $ErrorActionPreference = 'Stop' + $packageArgs = @('{{ replace(package_args, "'", "''") }}' -split '\s+' | Where-Object { $_ }) + + $metadataJson = & cargo {{_anvil_stable_toolchain_args}} metadata --no-deps --format-version 1 + if ($LASTEXITCODE -ne 0) { + Write-Error 'anvil miri: cargo metadata failed' + exit $LASTEXITCODE + } + try { + $metadata = $metadataJson | ConvertFrom-Json -ErrorAction Stop + } catch { + Write-Error "anvil miri: could not parse cargo metadata output: $($_.Exception.Message)" + exit 1 + } + + $workspaceMembers = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($member in $metadata.workspace_members) { + [void]$workspaceMembers.Add([string]$member) + } + $packageDirectories = @{} + $excludedPackages = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + $excludedPackageIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($package in $metadata.packages) { + $packageId = [string]$package.id + if (-not $workspaceMembers.Contains($packageId)) { continue } + $packageDirectories[$packageId] = Split-Path -Parent ([string]$package.manifest_path) + if ($package.metadata.anvil.miri.exclude -eq $true) { + [void]$excludedPackages.Add([string]$package.name) + [void]$excludedPackageIds.Add($packageId) + } + } + + if ($packageArgs -contains '--workspace') { + if ($workspaceMembers.Count -gt 0 -and $excludedPackageIds.Count -eq $workspaceMembers.Count) { + Write-Host 'anvil miri: all selected packages are excluded; nothing to test' + exit 0 + } + foreach ($packageName in $excludedPackages) { + $packageArgs += @('--exclude', $packageName) + } + } else { + $filtered = [System.Collections.Generic.List[string]]::new() + for ($i = 0; $i -lt $packageArgs.Count; $i++) { + if ($packageArgs[$i] -ne '--package') { + Write-Error "anvil miri: unexpected impact argument '$($packageArgs[$i])'" + exit 1 + } + if (($i + 1) -ge $packageArgs.Count) { + Write-Error 'anvil miri: --package is missing its package spec' + exit 1 + } + $packageSpec = $packageArgs[++$i] + $packageName = ($packageSpec -split '@', 2)[0] + if ($excludedPackages.Contains($packageName)) { continue } + $filtered.Add('--package') + $filtered.Add($packageSpec) + } + $packageArgs = $filtered.ToArray() + if ($packageArgs.Count -eq 0) { + Write-Host 'anvil miri: all selected packages are excluded; nothing to test' + exit 0 + } + } + + $logRoot = Join-Path ([IO.Path]::GetTempPath()) "anvil-miri-$PID-$([guid]::NewGuid())" + New-Item -ItemType Directory -Path $logRoot | Out-Null + try { + $artifactManifest = Join-Path $logRoot 'artifacts.jsonl' + $toolchain = '+{{ rust_nightly }}' + & cargo '+{{ rust_nightly }}' miri test --all-features --tests --no-run --message-format=json-render-diagnostics @packageArgs 1> $artifactManifest + $buildExitCode = $LASTEXITCODE + + $artifacts = [System.Collections.Generic.List[object]]::new() + $seenArtifacts = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($line in Get-Content -LiteralPath $artifactManifest) { + try { + $message = $line | ConvertFrom-Json -ErrorAction Stop + } catch { + Write-Error "anvil miri: could not parse Cargo JSON output: $($_.Exception.Message)" + exit 1 + } + if ($message.reason -eq 'compiler-message' -and $message.message.rendered) { + [Console]::Error.Write($message.message.rendered) + continue + } + if ($message.reason -ne 'compiler-artifact' -or -not $message.executable -or $message.profile.test -ne $true) { + continue + } + $artifactPath = [string]$message.executable + if (-not $seenArtifacts.Add($artifactPath)) { continue } + $packageId = [string]$message.package_id + if ($excludedPackageIds.Contains($packageId)) { continue } + if (-not $packageDirectories.ContainsKey($packageId)) { + Write-Error "anvil miri: package metadata not found for artifact package '$packageId'" + exit 1 + } + $targetName = [string]$message.target.name + $targetKinds = @($message.target.kind | ForEach-Object { [string]$_ }) + if ([string]::IsNullOrWhiteSpace($targetName) -or $targetKinds.Count -eq 0) { + Write-Error "anvil miri: target metadata not found for artifact '$artifactPath'" + exit 1 + } + $targetKind = $targetKinds -join '/' + $artifactFileName = Split-Path -Leaf $artifactPath + $artifacts.Add([pscustomobject]@{ + Path = $artifactPath + PackageId = $packageId + TargetName = $targetName + TargetKind = $targetKind + Label = "$packageId :: $targetKind $targetName ($artifactFileName)" + WorkingDirectory = $packageDirectories[$packageId] + }) + } + + if ($buildExitCode -ne 0) { exit $buildExitCode } + if ($artifacts.Count -eq 0) { + Write-Host 'anvil miri: Cargo produced no runnable test artifacts; nothing to test' + exit 0 + } + + $sysrootOutput = @(& cargo '+{{ rust_nightly }}' miri setup --print-sysroot) + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if ($sysrootOutput.Count -eq 0 -or -not $sysrootOutput[-1]) { + Write-Error 'anvil miri: cargo miri setup did not report a sysroot' + exit 1 + } + $env:MIRI_SYSROOT = $sysrootOutput[-1] + $toolchainSysroot = & rustc $toolchain --print sysroot + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($toolchainSysroot)) { + Write-Error "anvil miri: could not resolve the $toolchain toolchain sysroot" + exit 1 + } + $toolchainBin = Join-Path $toolchainSysroot.Trim() 'bin' + $runner = @( + 'cargo-miri.exe', 'cargo-miri', 'cargo-miri.ps1' | + ForEach-Object { Join-Path $toolchainBin $_ } | + Where-Object { Test-Path -LiteralPath $_ -PathType Leaf } + ) | Select-Object -First 1 + if (-not $runner) { + Write-Error "anvil miri: cargo-miri was not found under '$toolchainBin'" + exit 1 + } + # cargo-miri records MIRI_BE_RUSTC=host in each artifact. Keeping the + # same value in the standalone runner environment lets phase_runner + # suppress that recorded value before launching Miri for interpretation. + $env:MIRI_BE_RUSTC = 'host' + + $jobs = 0 + if ($env:ANVIL_MIRI_JOBS) { + if (-not [int]::TryParse($env:ANVIL_MIRI_JOBS, [ref]$jobs) -or $jobs -lt 1) { + Write-Error "anvil miri: ANVIL_MIRI_JOBS must be a positive integer, got '$($env:ANVIL_MIRI_JOBS)'" + exit 1 + } + } else { + $jobs = [Environment]::ProcessorCount + } + $jobs = [Math]::Min($jobs, $artifacts.Count) + + Write-Host "anvil miri: running $($artifacts.Count) test artifacts with $jobs concurrent process(es)" + + # Cargo normally runs each test executable from its owning package root. + # The direct cargo-miri invocation below restores that working directory. + $orderedArtifacts = @($artifacts | Sort-Object PackageId, TargetKind, TargetName, Path) + $work = for ($i = 0; $i -lt $orderedArtifacts.Count; $i++) { + [pscustomobject]@{ + Index = $i + Artifact = $orderedArtifacts[$i].Path + Label = $orderedArtifacts[$i].Label + WorkingDirectory = $orderedArtifacts[$i].WorkingDirectory + } + } + $results = @( + $work | ForEach-Object -Parallel { + $ErrorActionPreference = 'Stop' + $item = $_ + $log = Join-Path $using:logRoot "$($item.Index).log" + Write-Host "anvil miri: starting $($item.Label)" + Push-Location -LiteralPath $item.WorkingDirectory + try { + & $using:runner runner $item.Artifact *> $log + $exitCode = $LASTEXITCODE + } catch { + $_ | Out-String | Set-Content -LiteralPath $log + $exitCode = 1 + } finally { + Pop-Location + } + Write-Host "anvil miri: completed $($item.Label) (exit $exitCode)" + [pscustomobject]@{ + Index = $item.Index + Label = $item.Label + ExitCode = $exitCode + Log = $log + } + } -ThrottleLimit $jobs + ) + + # Parallel workers buffer into separate logs. Replaying those logs by + # stable artifact index keeps diagnostics isolated and deterministic. + foreach ($result in $results | Sort-Object Index) { + if ($env:GITHUB_ACTIONS -eq 'true') { + Write-Host "::group::Miri artifact $($result.Label)" + } elseif ($env:TF_BUILD -eq 'True') { + Write-Host "##[group]Miri artifact $($result.Label)" + } else { + Write-Host "`n=== Miri artifact $($result.Label) ===" + } + Get-Content -LiteralPath $result.Log | ForEach-Object { Write-Host $_ } + if ($env:GITHUB_ACTIONS -eq 'true') { + Write-Host '::endgroup::' + } elseif ($env:TF_BUILD -eq 'True') { + Write-Host '##[endgroup]' + } + } + + $failed = @($results | Where-Object ExitCode -ne 0) + if ($failed.Count -ne 0) { + [Console]::Error.WriteLine("anvil miri: failed artifacts: $($failed.Label -join ', ')") + exit 1 + } + } finally { + Remove-Item -LiteralPath $logRoot -Recurse -Force -ErrorAction SilentlyContinue + } + # Run Miri tests for affected workspace packages. [script("pwsh", "-NoProfile")] anvil-miri: anvil-miri-validate-prereqs anvil-impact @@ -43,7 +286,8 @@ anvil-miri: anvil-miri-validate-prereqs anvil-impact # whose tests are all `#[cfg_attr(miri, ignore)]` -- the canonical # opt-out for build-tooling / CLI crates), so the old nextest # `--no-tests=pass` workaround is no longer needed. - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @(if ($include) { -split $include } else { '--workspace' }) + $pkg = @(if ($include) { -split $include } else { '--workspace' }) + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # --- pr-runtime-analysis members --- diff --git a/crates/cargo-anvil/templates/justfiles/anvil/container.just b/crates/cargo-anvil/templates/justfiles/anvil/container.just index 6e3a91d8..82a98d51 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/container.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/container.just @@ -972,9 +972,10 @@ anvil-container *command: # The recipe contract's own inputs. These are read by generated checks -- # `anvil-pr-title` reads PR_TITLE, `_anvil-base-ref` reads BASE_REF and its - # CI equivalents, and `anvil-impact` reads ANVIL_IMPACT to decide whether to - # compute scoping, consume a downloaded cache, or skip -- so dropping them at - # the boundary makes the same command mean different things inside and out. + # CI equivalents, `anvil-impact` reads ANVIL_IMPACT to decide whether to + # compute scoping, consume a downloaded cache, or skip, and the Miri runner + # reads ANVIL_MIRI_JOBS to cap artifact workers -- so dropping them at the + # boundary makes the same command mean different things inside and out. # anvil-pr-title is the sharp case: with PR_TITLE unset it exits 0 with a # skip notice, so a title a native run rejects passes in a container and the # tier still reports green. ANVIL_IMPACT is the other: a CI group job exports @@ -987,7 +988,7 @@ anvil-container *command: foreach ($name in @( 'PR_TITLE', 'BASE_REF', 'GITHUB_BASE_REF', 'SYSTEM_PULLREQUEST_TARGETBRANCH', - 'ANVIL_IMPACT')) { + 'ANVIL_IMPACT', 'ANVIL_MIRI_JOBS')) { if ((Test-Path -LiteralPath "Env:$name") -and -not [string]::IsNullOrEmpty((Get-Item -LiteralPath "Env:$name").Value)) { $forwardedEnv += $name $runArgs += @('-e', $name) diff --git a/crates/cargo-anvil/tests/recipe_contracts.rs b/crates/cargo-anvil/tests/recipe_contracts.rs index 7b1f8220..487834ff 100644 --- a/crates/cargo-anvil/tests/recipe_contracts.rs +++ b/crates/cargo-anvil/tests/recipe_contracts.rs @@ -23,6 +23,8 @@ const BOLERO: &str = include_str!("../templates/justfiles/anvil/checks/bolero.ju const FMT: &str = include_str!("../templates/justfiles/anvil/checks/fmt.just"); const LLVM_COV: &str = include_str!("../templates/justfiles/anvil/checks/llvm-cov.just"); const LOOM: &str = include_str!("../templates/justfiles/anvil/checks/loom.just"); +const MIRI: &str = include_str!("../templates/justfiles/anvil/checks/miri.just"); +const MIRI_TREE_BORROWS: &str = include_str!("../templates/justfiles/anvil/checks/miri-tree-borrows.just"); const MSRV_TEST: &str = include_str!("../templates/justfiles/anvil/checks/msrv-test.just"); const SEMVER: &str = include_str!("../templates/justfiles/anvil/checks/semver-check.just"); const EXTERNAL_TYPES: &str = include_str!("../templates/justfiles/anvil/checks/external-types.just"); @@ -35,6 +37,10 @@ const CONTAINER: &str = include_str!("../templates/justfiles/anvil/container.jus // Any nonzero value works; naming it prevents tests from implying an external exit-code contract. const ARBITRARY_FAILURE_EXIT: &str = "23"; +// The Miri fixture mirrors the production protocol rather than executing Rust: +// fake Cargo supplies workspace metadata and compiler-artifact JSON, fake rustc +// identifies the pinned toolchain sysroot, and cargo-miri under that sysroot +// records the environment and working directory restored by the direct runner. #[test] fn regeneration_check_runs_on_every_pull_request() { assert!(REGENERATE_WORKFLOW.contains("pull_request: {}")); @@ -101,6 +107,14 @@ if ($args -contains 'metadata') { } else { [System.IO.Path]::Combine($root, 'Cargo.toml') } + $packageMetadata = [pscustomobject]@{ + 'coverage-gate' = [pscustomobject]@{ 'min-lines-percent' = 0 } + } + if ($env:FAKE_MIRI_EXCLUDE) { + $packageMetadata | Add-Member -NotePropertyName anvil -NotePropertyValue ( + [pscustomobject]@{ miri = [pscustomobject]@{ exclude = $true } } + ) + } $packages = @( [pscustomobject]@{ name = $packageName @@ -114,9 +128,7 @@ if ($args -contains 'metadata') { } else { $null } - metadata = [pscustomobject]@{ - 'coverage-gate' = [pscustomobject]@{ 'min-lines-percent' = 0 } - } + metadata = $packageMetadata } ) if ($env:FAKE_SECOND_PACKAGE_NAME) { @@ -125,6 +137,12 @@ if ($args -contains 'metadata') { } else { $env:FAKE_PACKAGE_DIR_LEAF } + $secondMetadata = [pscustomobject]@{} + if ($env:FAKE_SECOND_MIRI_EXCLUDE) { + $secondMetadata | Add-Member -NotePropertyName anvil -NotePropertyValue ( + [pscustomobject]@{ miri = [pscustomobject]@{ exclude = $true } } + ) + } $packages += [pscustomobject]@{ name = $env:FAKE_SECOND_PACKAGE_NAME version = '0.1.0' @@ -132,7 +150,7 @@ if ($args -contains 'metadata') { manifest_path = [System.IO.Path]::Combine($root, 'nested', $secondDirLeaf, 'Cargo.toml') targets = @([pscustomobject]@{ name = $env:FAKE_SECOND_PACKAGE_NAME; kind = @('lib') }) publish = $null - metadata = [pscustomobject]@{} + metadata = $secondMetadata } } if ($env:FAKE_THIRD_PACKAGE_NAME) { @@ -173,6 +191,77 @@ if ($args -contains 'metadata') { $metadata | ConvertTo-Json -Depth 8 -Compress exit 0 } +if ($args -contains 'miri') { + if ($args -contains 'setup') { + Write-Output ([System.IO.Path]::Combine($env:FAKE_WORKSPACE_ROOT, 'fake-miri-sysroot')) + exit [int]$env:FAKE_MIRI_SETUP_EXIT + } + if ($args -contains '--no-run') { + if ($env:FAKE_MIRI_INVALID_JSON) { + Write-Output '{invalid cargo json' + exit 0 + } + $artifactRoot = [System.IO.Path]::Combine($env:FAKE_WORKSPACE_ROOT, 'fake artifacts') + [System.IO.Directory]::CreateDirectory($artifactRoot) | Out-Null + $definitions = @() + if ($env:FAKE_MIRI_ARTIFACTS) { + $definitions = @($env:FAKE_MIRI_ARTIFACTS | ConvertFrom-Json) + } + foreach ($definition in $definitions) { + $artifactDirectory = if ($definition.artifact_dir) { + [System.IO.Path]::Combine($artifactRoot, [string]$definition.artifact_dir) + } else { + $artifactRoot + } + [System.IO.Directory]::CreateDirectory($artifactDirectory) | Out-Null + $path = [System.IO.Path]::Combine($artifactDirectory, [string]$definition.name) + Set-Content -LiteralPath $path -Value '{}' + [pscustomobject]@{ + reason = 'compiler-artifact' + package_id = [string]$definition.package_id + executable = $path + profile = [pscustomobject]@{ test = [bool]$definition.test } + target = [pscustomobject]@{ + name = if ($definition.target_name) { + [string]$definition.target_name + } else { + [string]$definition.name + } + kind = @(if ($definition.target_kind) { + [string]$definition.target_kind + } else { + 'test' + }) + } + } | ConvertTo-Json -Depth 4 -Compress + if ($definition.duplicate) { + [pscustomobject]@{ + reason = 'compiler-artifact' + package_id = [string]$definition.package_id + executable = $path + profile = [pscustomobject]@{ test = [bool]$definition.test } + target = [pscustomobject]@{ + name = if ($definition.target_name) { + [string]$definition.target_name + } else { + [string]$definition.name + } + kind = @(if ($definition.target_kind) { + [string]$definition.target_kind + } else { + 'test' + }) + } + } | ConvertTo-Json -Depth 4 -Compress + } + } + [pscustomobject]@{ + reason = 'build-finished' + success = ([int]$env:FAKE_MIRI_BUILD_EXIT -eq 0) + } | ConvertTo-Json -Compress + exit [int]$env:FAKE_MIRI_BUILD_EXIT + } +} if ($args -contains 'semver-checks') { if ($env:FAKE_SEMVER_OUTPUT) { Write-Output $env:FAKE_SEMVER_OUTPUT } exit [int]$env:FAKE_SEMVER_EXIT @@ -247,6 +336,38 @@ fn fixture(imports: &[(&str, &str)], dependency_recipes: &[&str]) -> TempDir { fs::create_dir_all(&bin).unwrap(); write(&bin.join("cargo.ps1"), FAKE_CARGO_PS1); write(&bin.join("git.ps1"), "exit 0\n"); + write( + &bin.join("rustc.ps1"), + r" +if ($args -contains 'sysroot') { + Write-Output ([System.IO.Path]::Combine($env:FAKE_WORKSPACE_ROOT, 'fake-toolchain')) + exit 0 +} +exit 1 +", + ); + let fake_toolchain_bin = tmp.path().join("fake-toolchain/bin"); + fs::create_dir_all(&fake_toolchain_bin).unwrap(); + write( + &fake_toolchain_bin.join("cargo-miri.ps1"), + r#" +$artifact = [System.IO.Path]::GetFileName([string]$args[-1]) +$prefix = "$($env:FAKE_MIRI_RUN_LOG).$artifact" +(Get-Location).Path | Set-Content -LiteralPath "$prefix.cwd" +$env:MIRI_SYSROOT | Set-Content -LiteralPath "$prefix.sysroot" +$env:MIRI_BE_RUSTC | Set-Content -LiteralPath "$prefix.miri-be-rustc" +$env:MIRIFLAGS | Set-Content -LiteralPath "$prefix.miriflags" +$env:RUSTFLAGS | Set-Content -LiteralPath "$prefix.rustflags" +(Get-Date).ToUniversalTime().Ticks | Set-Content -LiteralPath "$prefix.start" +if ($env:FAKE_MIRI_SLEEP_MS) { + Start-Sleep -Milliseconds ([int]$env:FAKE_MIRI_SLEEP_MS) +} +Write-Output "miri output: $artifact" +(Get-Date).ToUniversalTime().Ticks | Set-Content -LiteralPath "$prefix.end" +if ($artifact -like '*fail*') { exit 9 } +exit 0 +"#, + ); tmp } @@ -265,15 +386,12 @@ fn run_just(root: &Path, arguments: &[&str], environment: &[(&str, &OsStr)]) -> .current_dir(root); command.env("PATH", path_with_fake_bin(root)); command.env("FAKE_WORKSPACE_ROOT", root); - // A fixture is a scratch workspace, so it must not inherit the impact - // scoping of whatever invoked the test suite. A CI group job exports - // `ANVIL_IMPACT=consume` and downloads a cache into the real repository; - // inherited into a temp directory that has no cache, `anvil-impact` fails - // hard and takes the recipe under test with it. `ANVIL_INCLUDE_*` is the - // same hazard one level down: a leg whose scope resolved to `--skip` would - // silently short-circuit the recipe before it did anything. A test that - // cares about either value passes it explicitly below. + // A fixture is a scratch workspace, so it must not inherit impact scoping + // or output-backend markers from the process running the test suite. Tests + // that exercise those contracts pass the relevant values explicitly. command.env_remove("ANVIL_IMPACT"); + command.env_remove("GITHUB_ACTIONS"); + command.env_remove("TF_BUILD"); for key in std::env::vars_os().map(|(key, _)| key) { if key.to_string_lossy().starts_with("ANVIL_INCLUDE_") { command.env_remove(key); @@ -294,6 +412,328 @@ fn assert_failed(output: &Output, context: &str) { ); } +#[test] +fn miri_runner_filters_artifacts_and_runs_in_parallel() { + if !tools_available() { + return; + } + let tmp = fixture( + &[("miri.just", MIRI)], + &[ + "anvil-component-nightly-miri-validate-prereqs", + "anvil-component-nightly-rust-src-validate-prereqs", + "anvil-component-nightly-miri-install", + "anvil-component-nightly-rust-src-install", + "anvil-impact", + ], + ); + let nested = tmp.path().join("nested/other-package"); + fs::create_dir_all(&nested).unwrap(); + write( + &nested.join("Cargo.toml"), + "[package]\nname = \"other-package\"\nversion = \"0.1.0\"\n", + ); + let cargo_log = tmp.path().join("cargo.log"); + let run_log = tmp.path().join("miri-run"); + let artifacts = r#"[ + {"name":"zeta-test","package_id":"fixture 0.1.0","target_name":"zeta","target_kind":"test","test":true}, + {"name":"alpha-test","package_id":"fixture 0.1.0","target_name":"alpha","target_kind":"test","test":true,"duplicate":true}, + {"name":"ordinary-bin","package_id":"fixture 0.1.0","test":false}, + {"name":"excluded-test","package_id":"other-package 0.1.0","test":true} + ]"#; + let output = run_just( + tmp.path(), + &["_anvil-miri-test", "--workspace"], + &[ + ("ANVIL_MIRI_JOBS", OsStr::new("2")), + ("FAKE_CARGO_LOG", cargo_log.as_os_str()), + ("FAKE_MIRI_ARTIFACTS", OsStr::new(artifacts)), + ("FAKE_MIRI_RUN_LOG", run_log.as_os_str()), + ("FAKE_MIRI_SLEEP_MS", OsStr::new("300")), + ("FAKE_SECOND_PACKAGE_NAME", OsStr::new("other-package")), + ("FAKE_SECOND_PACKAGE_DIR_LEAF", OsStr::new("other-package")), + ("FAKE_SECOND_MIRI_EXCLUDE", OsStr::new("1")), + ], + ); + + assert!( + output.status.success(), + "parallel Miri runner should succeed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let cargo_calls = fs::read_to_string(cargo_log).unwrap(); + assert!( + cargo_calls.contains( + "miri test --all-features --tests --no-run --message-format=json-render-diagnostics --workspace --exclude other-package" + ), + "workspace Miri compilation must exclude metadata-opted-out packages:\n{cargo_calls}" + ); + assert!(run_log.with_extension("alpha-test.start").is_file()); + assert!(run_log.with_extension("zeta-test.start").is_file()); + assert!(!run_log.with_extension("ordinary-bin.start").exists()); + assert!(!run_log.with_extension("excluded-test.start").exists()); + + let recorded_cwd = fs::read_to_string(run_log.with_extension("alpha-test.cwd")).unwrap(); + let expected_cwd = fs::canonicalize(tmp.path()).unwrap(); + assert_eq!(fs::canonicalize(recorded_cwd.trim()).unwrap(), expected_cwd); + assert_eq!( + fs::read_to_string(run_log.with_extension("alpha-test.sysroot")).unwrap().trim(), + tmp.path().join("fake-miri-sysroot").to_str().unwrap() + ); + assert_eq!( + fs::read_to_string(run_log.with_extension("alpha-test.miri-be-rustc")) + .unwrap() + .trim(), + "host" + ); + let alpha_started = fs::read_to_string(run_log.with_extension("alpha-test.start")) + .unwrap() + .trim() + .parse::() + .unwrap(); + let zeta_started = fs::read_to_string(run_log.with_extension("zeta-test.start")) + .unwrap() + .trim() + .parse::() + .unwrap(); + let alpha_ended = fs::read_to_string(run_log.with_extension("alpha-test.end")) + .unwrap() + .trim() + .parse::() + .unwrap(); + let zeta_ended = fs::read_to_string(run_log.with_extension("zeta-test.end")) + .unwrap() + .trim() + .parse::() + .unwrap(); + assert!( + alpha_started < zeta_ended && zeta_started < alpha_ended, + "two workers should overlap: alpha {alpha_started}..{alpha_ended}, zeta {zeta_started}..{zeta_ended}" + ); + + let stdout = String::from_utf8_lossy(&output.stdout); + let alpha_group = stdout.find("=== Miri artifact fixture 0.1.0 :: test alpha").unwrap(); + let zeta_group = stdout.find("=== Miri artifact fixture 0.1.0 :: test zeta").unwrap(); + assert!(alpha_group < zeta_group, "artifact logs must replay deterministically"); +} + +#[test] +fn miri_runner_labels_same_named_artifacts_with_package_identity() { + if !tools_available() { + return; + } + let tmp = fixture( + &[("miri.just", MIRI)], + &[ + "anvil-component-nightly-miri-validate-prereqs", + "anvil-component-nightly-rust-src-validate-prereqs", + "anvil-component-nightly-miri-install", + "anvil-component-nightly-rust-src-install", + "anvil-impact", + ], + ); + let other_package = tmp.path().join("nested/other-package"); + fs::create_dir_all(&other_package).unwrap(); + write( + &other_package.join("Cargo.toml"), + "[package]\nname = \"other-package\"\nversion = \"0.1.0\"\n", + ); + let run_log = tmp.path().join("same-name"); + let artifacts = r#"[ + {"name":"shared-test","artifact_dir":"fixture","package_id":"fixture 0.1.0","target_name":"shared","target_kind":"test","test":true}, + {"name":"shared-test","artifact_dir":"other","package_id":"other-package 0.1.0","target_name":"shared","target_kind":"test","test":true} + ]"#; + let output = run_just( + tmp.path(), + &["_anvil-miri-test", "--workspace"], + &[ + ("FAKE_MIRI_ARTIFACTS", OsStr::new(artifacts)), + ("FAKE_MIRI_RUN_LOG", run_log.as_os_str()), + ("ANVIL_MIRI_JOBS", OsStr::new("1")), + ("FAKE_SECOND_PACKAGE_NAME", OsStr::new("other-package")), + ("FAKE_SECOND_PACKAGE_DIR_LEAF", OsStr::new("other-package")), + ], + ); + + assert!( + output.status.success(), + "same-named Miri artifacts should both run:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("Miri artifact fixture 0.1.0 :: test shared (shared-test)")); + assert!(stdout.contains("Miri artifact other-package 0.1.0 :: test shared (shared-test)")); +} + +#[test] +fn miri_runner_preserves_profile_flags_and_impact_filtering() { + if !tools_available() { + return; + } + let tmp = fixture( + &[("miri.just", MIRI), ("miri-tree-borrows.just", MIRI_TREE_BORROWS)], + &[ + "anvil-component-nightly-miri-validate-prereqs", + "anvil-component-nightly-rust-src-validate-prereqs", + "anvil-component-nightly-miri-install", + "anvil-component-nightly-rust-src-install", + "anvil-impact", + ], + ); + let justfile_path = tmp.path().join("Justfile"); + let mut justfile = fs::read_to_string(&justfile_path).unwrap(); + justfile.push_str( + r#" +[script("pwsh", "-NoProfile")] +_anvil-impact-include tier: + Write-Output $env:FAKE_INCLUDE +"#, + ); + write(&justfile_path, &justfile); + let run_log = tmp.path().join("miri-profile"); + let cargo_log = tmp.path().join("miri-profile-cargo.log"); + let artifacts = r#"[{"name":"profile-test","package_id":"fixture 0.1.0","test":true}]"#; + let output = run_just( + tmp.path(), + &["anvil-miri-tree-borrows"], + &[ + ("FAKE_INCLUDE", OsStr::new("--package fixture@0.1.0 --package excluded@0.1.0")), + ("FAKE_CARGO_LOG", cargo_log.as_os_str()), + ("FAKE_MIRI_ARTIFACTS", OsStr::new(artifacts)), + ("FAKE_MIRI_RUN_LOG", run_log.as_os_str()), + ("FAKE_SECOND_PACKAGE_NAME", OsStr::new("excluded")), + ("FAKE_SECOND_MIRI_EXCLUDE", OsStr::new("1")), + ], + ); + + assert!( + output.status.success(), + "Tree Borrows profile should delegate to the shared runner:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + fs::read_to_string(run_log.with_extension("profile-test.miriflags")) + .unwrap() + .contains("-Zmiri-tree-borrows") + ); + assert!( + fs::read_to_string(run_log.with_extension("profile-test.rustflags")) + .unwrap() + .contains("--cfg miri_tree_borrows") + ); + let cargo_calls = fs::read_to_string(cargo_log).unwrap(); + let build_call = cargo_calls + .lines() + .find(|call| call.contains("miri test")) + .expect("the profile must compile its selected Miri targets"); + assert!( + build_call.contains("--package fixture@0.1.0"), + "impact-selected included packages must be forwarded:\n{build_call}" + ); + assert!( + !build_call.contains("excluded@0.1.0"), + "impact-selected opted-out packages must be filtered:\n{build_call}" + ); +} + +#[test] +fn miri_runner_handles_no_work_and_aggregates_failures() { + if !tools_available() { + return; + } + let tmp = fixture( + &[("miri.just", MIRI)], + &[ + "anvil-component-nightly-miri-validate-prereqs", + "anvil-component-nightly-rust-src-validate-prereqs", + "anvil-component-nightly-miri-install", + "anvil-component-nightly-rust-src-install", + "anvil-impact", + ], + ); + let no_artifacts = run_just(tmp.path(), &["_anvil-miri-test", "--workspace"], &[]); + assert!( + no_artifacts.status.success(), + "a selected package set with no test artifacts should succeed:\n{}", + String::from_utf8_lossy(&no_artifacts.stderr) + ); + assert!(String::from_utf8_lossy(&no_artifacts.stdout).contains("no runnable test artifacts")); + + let all_excluded = run_just( + tmp.path(), + &["_anvil-miri-test", "--package", "fixture@0.1.0"], + &[("FAKE_MIRI_EXCLUDE", OsStr::new("1"))], + ); + assert!(all_excluded.status.success()); + assert!(String::from_utf8_lossy(&all_excluded.stdout).contains("all selected packages are excluded")); + + let workspace_cargo_log = tmp.path().join("all-excluded-workspace-cargo.log"); + let all_workspace_excluded = run_just( + tmp.path(), + &["_anvil-miri-test", "--workspace"], + &[ + ("FAKE_CARGO_LOG", workspace_cargo_log.as_os_str()), + ("FAKE_MIRI_EXCLUDE", OsStr::new("1")), + ], + ); + assert!(all_workspace_excluded.status.success()); + assert!(String::from_utf8_lossy(&all_workspace_excluded.stdout).contains("all selected packages are excluded")); + assert!( + !fs::read_to_string(workspace_cargo_log).unwrap().contains("miri test"), + "an all-excluded workspace must stop before asking Cargo to select no packages" + ); + + let run_log = tmp.path().join("miri-failure"); + let artifacts = r#"[ + {"name":"fail-test","package_id":"fixture 0.1.0","test":true}, + {"name":"pass-test","package_id":"fixture 0.1.0","test":true} + ]"#; + let failed = run_just( + tmp.path(), + &["_anvil-miri-test", "--workspace"], + &[ + ("FAKE_MIRI_ARTIFACTS", OsStr::new(artifacts)), + ("FAKE_MIRI_RUN_LOG", run_log.as_os_str()), + ("ANVIL_MIRI_JOBS", OsStr::new("2")), + ("TF_BUILD", OsStr::new("True")), + ], + ); + assert_failed(&failed, "one failed Miri artifact"); + let stdout = String::from_utf8_lossy(&failed.stdout); + let stderr = String::from_utf8_lossy(&failed.stderr); + assert!(stdout.contains("miri output: fail-test")); + assert!(stdout.contains("miri output: pass-test")); + assert!(stdout.contains("##[group]Miri artifact fixture 0.1.0 :: test fail-test (fail-test)")); + assert!(stdout.contains("##[endgroup]")); + assert!(stderr.contains("failed artifacts: fixture 0.1.0 :: test fail-test (fail-test)")); + + let invalid_jobs = run_just( + tmp.path(), + &["_anvil-miri-test", "--workspace"], + &[ + ( + "FAKE_MIRI_ARTIFACTS", + OsStr::new(r#"[{"name":"jobs-test","package_id":"fixture 0.1.0","test":true}]"#), + ), + ("FAKE_MIRI_RUN_LOG", run_log.as_os_str()), + ("ANVIL_MIRI_JOBS", OsStr::new("0")), + ], + ); + assert_failed(&invalid_jobs, "invalid ANVIL_MIRI_JOBS"); + assert!(String::from_utf8_lossy(&invalid_jobs.stderr).contains("anvil miri: ANVIL_MIRI_JOBS must be a positive integer")); + + let invalid_json = run_just( + tmp.path(), + &["_anvil-miri-test", "--workspace"], + &[("FAKE_MIRI_INVALID_JSON", OsStr::new("1"))], + ); + assert_failed(&invalid_json, "malformed Cargo JSON"); + assert!(String::from_utf8_lossy(&invalid_json.stderr).contains("could not parse Cargo JSON output")); +} + #[test] fn stable_command_leaves_environment_toolchain_selection_native() { if !tools_available() { diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap index e8063b70..95e80fad 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap @@ -2616,7 +2616,7 @@ anvil-miri-race-coverage: anvil-miri-race-coverage-validate-prereqs anvil-impact $env:MIRIFLAGS = "-Zmiri-many-seeds=$low..$high $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_race_coverage $($env:RUSTFLAGS)".Trim() Write-Host "anvil-miri-race-coverage: seed window $low..$high (day $day)" - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Install prerequisites for the `anvil-miri-race-coverage` recipe. @@ -2645,7 +2645,7 @@ anvil-miri-strict-provenance: anvil-miri-strict-provenance-validate-prereqs anvi $pkg = @(if ($include) { -split $include } else { '--workspace' }) $env:MIRIFLAGS = "-Zmiri-strict-provenance $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_strict_provenance $($env:RUSTFLAGS)".Trim() - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Install prerequisites for the `anvil-miri-strict-provenance` recipe. @@ -2688,7 +2688,7 @@ anvil-miri-tree-borrows: anvil-miri-tree-borrows-validate-prereqs anvil-impact $pkg = @(if ($include) { -split $include } else { '--workspace' }) $env:MIRIFLAGS = "-Zmiri-tree-borrows $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_tree_borrows $($env:RUSTFLAGS)".Trim() - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # The three nightly-miri profiles share the same toolchain prereqs @@ -2719,6 +2719,249 @@ anvil-miri-tree-borrows-validate-prereqs: anvil-component-nightly-miri-validate- # scheduled/full tiers (which force ANVIL_IMPACT=off) or a dirty local tree # (which widens for safety). +# Compile all selected packages together, then run their Miri test artifacts in +# parallel. The single Cargo invocation preserves workspace feature unification, +# while artifact-level parallelism uses host logical processors that +# `cargo miri test` otherwise leaves idle. +# +# Packages can opt their own test targets out through: +# +# [package.metadata.anvil.miri] +# exclude = true +# +# They can still compile as dependencies of selected consumers. Workspace runs +# translate the metadata to `--exclude`; impact-scoped runs remove matching +# `--package name@version` pairs before compilation. +# +# ANVIL_MIRI_JOBS overrides the default of one artifact worker per logical processor. + +# Compile selected packages once and run their Miri test artifacts concurrently. +[script("pwsh", "-NoProfile")] +_anvil-miri-test *package_args: + $ErrorActionPreference = 'Stop' + $packageArgs = @('{{ replace(package_args, "'", "''") }}' -split '\s+' | Where-Object { $_ }) + + $metadataJson = & cargo {{_anvil_stable_toolchain_args}} metadata --no-deps --format-version 1 + if ($LASTEXITCODE -ne 0) { + Write-Error 'anvil miri: cargo metadata failed' + exit $LASTEXITCODE + } + try { + $metadata = $metadataJson | ConvertFrom-Json -ErrorAction Stop + } catch { + Write-Error "anvil miri: could not parse cargo metadata output: $($_.Exception.Message)" + exit 1 + } + + $workspaceMembers = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($member in $metadata.workspace_members) { + [void]$workspaceMembers.Add([string]$member) + } + $packageDirectories = @{} + $excludedPackages = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + $excludedPackageIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($package in $metadata.packages) { + $packageId = [string]$package.id + if (-not $workspaceMembers.Contains($packageId)) { continue } + $packageDirectories[$packageId] = Split-Path -Parent ([string]$package.manifest_path) + if ($package.metadata.anvil.miri.exclude -eq $true) { + [void]$excludedPackages.Add([string]$package.name) + [void]$excludedPackageIds.Add($packageId) + } + } + + if ($packageArgs -contains '--workspace') { + if ($workspaceMembers.Count -gt 0 -and $excludedPackageIds.Count -eq $workspaceMembers.Count) { + Write-Host 'anvil miri: all selected packages are excluded; nothing to test' + exit 0 + } + foreach ($packageName in $excludedPackages) { + $packageArgs += @('--exclude', $packageName) + } + } else { + $filtered = [System.Collections.Generic.List[string]]::new() + for ($i = 0; $i -lt $packageArgs.Count; $i++) { + if ($packageArgs[$i] -ne '--package') { + Write-Error "anvil miri: unexpected impact argument '$($packageArgs[$i])'" + exit 1 + } + if (($i + 1) -ge $packageArgs.Count) { + Write-Error 'anvil miri: --package is missing its package spec' + exit 1 + } + $packageSpec = $packageArgs[++$i] + $packageName = ($packageSpec -split '@', 2)[0] + if ($excludedPackages.Contains($packageName)) { continue } + $filtered.Add('--package') + $filtered.Add($packageSpec) + } + $packageArgs = $filtered.ToArray() + if ($packageArgs.Count -eq 0) { + Write-Host 'anvil miri: all selected packages are excluded; nothing to test' + exit 0 + } + } + + $logRoot = Join-Path ([IO.Path]::GetTempPath()) "anvil-miri-$PID-$([guid]::NewGuid())" + New-Item -ItemType Directory -Path $logRoot | Out-Null + try { + $artifactManifest = Join-Path $logRoot 'artifacts.jsonl' + $toolchain = '+{{ rust_nightly }}' + & cargo '+{{ rust_nightly }}' miri test --all-features --tests --no-run --message-format=json-render-diagnostics @packageArgs 1> $artifactManifest + $buildExitCode = $LASTEXITCODE + + $artifacts = [System.Collections.Generic.List[object]]::new() + $seenArtifacts = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($line in Get-Content -LiteralPath $artifactManifest) { + try { + $message = $line | ConvertFrom-Json -ErrorAction Stop + } catch { + Write-Error "anvil miri: could not parse Cargo JSON output: $($_.Exception.Message)" + exit 1 + } + if ($message.reason -eq 'compiler-message' -and $message.message.rendered) { + [Console]::Error.Write($message.message.rendered) + continue + } + if ($message.reason -ne 'compiler-artifact' -or -not $message.executable -or $message.profile.test -ne $true) { + continue + } + $artifactPath = [string]$message.executable + if (-not $seenArtifacts.Add($artifactPath)) { continue } + $packageId = [string]$message.package_id + if ($excludedPackageIds.Contains($packageId)) { continue } + if (-not $packageDirectories.ContainsKey($packageId)) { + Write-Error "anvil miri: package metadata not found for artifact package '$packageId'" + exit 1 + } + $targetName = [string]$message.target.name + $targetKinds = @($message.target.kind | ForEach-Object { [string]$_ }) + if ([string]::IsNullOrWhiteSpace($targetName) -or $targetKinds.Count -eq 0) { + Write-Error "anvil miri: target metadata not found for artifact '$artifactPath'" + exit 1 + } + $targetKind = $targetKinds -join '/' + $artifactFileName = Split-Path -Leaf $artifactPath + $artifacts.Add([pscustomobject]@{ + Path = $artifactPath + PackageId = $packageId + TargetName = $targetName + TargetKind = $targetKind + Label = "$packageId :: $targetKind $targetName ($artifactFileName)" + WorkingDirectory = $packageDirectories[$packageId] + }) + } + + if ($buildExitCode -ne 0) { exit $buildExitCode } + if ($artifacts.Count -eq 0) { + Write-Host 'anvil miri: Cargo produced no runnable test artifacts; nothing to test' + exit 0 + } + + $sysrootOutput = @(& cargo '+{{ rust_nightly }}' miri setup --print-sysroot) + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if ($sysrootOutput.Count -eq 0 -or -not $sysrootOutput[-1]) { + Write-Error 'anvil miri: cargo miri setup did not report a sysroot' + exit 1 + } + $env:MIRI_SYSROOT = $sysrootOutput[-1] + $toolchainSysroot = & rustc $toolchain --print sysroot + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($toolchainSysroot)) { + Write-Error "anvil miri: could not resolve the $toolchain toolchain sysroot" + exit 1 + } + $toolchainBin = Join-Path $toolchainSysroot.Trim() 'bin' + $runner = @( + 'cargo-miri.exe', 'cargo-miri', 'cargo-miri.ps1' | + ForEach-Object { Join-Path $toolchainBin $_ } | + Where-Object { Test-Path -LiteralPath $_ -PathType Leaf } + ) | Select-Object -First 1 + if (-not $runner) { + Write-Error "anvil miri: cargo-miri was not found under '$toolchainBin'" + exit 1 + } + # cargo-miri records MIRI_BE_RUSTC=host in each artifact. Keeping the + # same value in the standalone runner environment lets phase_runner + # suppress that recorded value before launching Miri for interpretation. + $env:MIRI_BE_RUSTC = 'host' + + $jobs = 0 + if ($env:ANVIL_MIRI_JOBS) { + if (-not [int]::TryParse($env:ANVIL_MIRI_JOBS, [ref]$jobs) -or $jobs -lt 1) { + Write-Error "anvil miri: ANVIL_MIRI_JOBS must be a positive integer, got '$($env:ANVIL_MIRI_JOBS)'" + exit 1 + } + } else { + $jobs = [Environment]::ProcessorCount + } + $jobs = [Math]::Min($jobs, $artifacts.Count) + + Write-Host "anvil miri: running $($artifacts.Count) test artifacts with $jobs concurrent process(es)" + + # Cargo normally runs each test executable from its owning package root. + # The direct cargo-miri invocation below restores that working directory. + $orderedArtifacts = @($artifacts | Sort-Object PackageId, TargetKind, TargetName, Path) + $work = for ($i = 0; $i -lt $orderedArtifacts.Count; $i++) { + [pscustomobject]@{ + Index = $i + Artifact = $orderedArtifacts[$i].Path + Label = $orderedArtifacts[$i].Label + WorkingDirectory = $orderedArtifacts[$i].WorkingDirectory + } + } + $results = @( + $work | ForEach-Object -Parallel { + $ErrorActionPreference = 'Stop' + $item = $_ + $log = Join-Path $using:logRoot "$($item.Index).log" + Write-Host "anvil miri: starting $($item.Label)" + Push-Location -LiteralPath $item.WorkingDirectory + try { + & $using:runner runner $item.Artifact *> $log + $exitCode = $LASTEXITCODE + } catch { + $_ | Out-String | Set-Content -LiteralPath $log + $exitCode = 1 + } finally { + Pop-Location + } + Write-Host "anvil miri: completed $($item.Label) (exit $exitCode)" + [pscustomobject]@{ + Index = $item.Index + Label = $item.Label + ExitCode = $exitCode + Log = $log + } + } -ThrottleLimit $jobs + ) + + # Parallel workers buffer into separate logs. Replaying those logs by + # stable artifact index keeps diagnostics isolated and deterministic. + foreach ($result in $results | Sort-Object Index) { + if ($env:GITHUB_ACTIONS -eq 'true') { + Write-Host "::group::Miri artifact $($result.Label)" + } elseif ($env:TF_BUILD -eq 'True') { + Write-Host "##[group]Miri artifact $($result.Label)" + } else { + Write-Host "`n=== Miri artifact $($result.Label) ===" + } + Get-Content -LiteralPath $result.Log | ForEach-Object { Write-Host $_ } + if ($env:GITHUB_ACTIONS -eq 'true') { + Write-Host '::endgroup::' + } elseif ($env:TF_BUILD -eq 'True') { + Write-Host '##[endgroup]' + } + } + + $failed = @($results | Where-Object ExitCode -ne 0) + if ($failed.Count -ne 0) { + [Console]::Error.WriteLine("anvil miri: failed artifacts: $($failed.Label -join ', ')") + exit 1 + } + } finally { + Remove-Item -LiteralPath $logRoot -Recurse -Force -ErrorAction SilentlyContinue + } + # Run Miri tests for affected workspace packages. [script("pwsh", "-NoProfile")] anvil-miri: anvil-miri-validate-prereqs anvil-impact @@ -2750,7 +2993,8 @@ anvil-miri: anvil-miri-validate-prereqs anvil-impact # whose tests are all `#[cfg_attr(miri, ignore)]` -- the canonical # opt-out for build-tooling / CLI crates), so the old nextest # `--no-tests=pass` workaround is no longer needed. - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @(if ($include) { -split $include } else { '--workspace' }) + $pkg = @(if ($include) { -split $include } else { '--workspace' }) + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # --- pr-runtime-analysis members --- @@ -4464,9 +4708,10 @@ anvil-container *command: # The recipe contract's own inputs. These are read by generated checks -- # `anvil-pr-title` reads PR_TITLE, `_anvil-base-ref` reads BASE_REF and its - # CI equivalents, and `anvil-impact` reads ANVIL_IMPACT to decide whether to - # compute scoping, consume a downloaded cache, or skip -- so dropping them at - # the boundary makes the same command mean different things inside and out. + # CI equivalents, `anvil-impact` reads ANVIL_IMPACT to decide whether to + # compute scoping, consume a downloaded cache, or skip, and the Miri runner + # reads ANVIL_MIRI_JOBS to cap artifact workers -- so dropping them at the + # boundary makes the same command mean different things inside and out. # anvil-pr-title is the sharp case: with PR_TITLE unset it exits 0 with a # skip notice, so a title a native run rejects passes in a container and the # tier still reports green. ANVIL_IMPACT is the other: a CI group job exports @@ -4479,7 +4724,7 @@ anvil-container *command: foreach ($name in @( 'PR_TITLE', 'BASE_REF', 'GITHUB_BASE_REF', 'SYSTEM_PULLREQUEST_TARGETBRANCH', - 'ANVIL_IMPACT')) { + 'ANVIL_IMPACT', 'ANVIL_MIRI_JOBS')) { if ((Test-Path -LiteralPath "Env:$name") -and -not [string]::IsNullOrEmpty((Get-Item -LiteralPath "Env:$name").Value)) { $forwardedEnv += $name $runArgs += @('-e', $name) diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index 45d27caf..c64e4db5 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -2602,7 +2602,7 @@ anvil-miri-race-coverage: anvil-miri-race-coverage-validate-prereqs anvil-impact $env:MIRIFLAGS = "-Zmiri-many-seeds=$low..$high $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_race_coverage $($env:RUSTFLAGS)".Trim() Write-Host "anvil-miri-race-coverage: seed window $low..$high (day $day)" - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Install prerequisites for the `anvil-miri-race-coverage` recipe. @@ -2631,7 +2631,7 @@ anvil-miri-strict-provenance: anvil-miri-strict-provenance-validate-prereqs anvi $pkg = @(if ($include) { -split $include } else { '--workspace' }) $env:MIRIFLAGS = "-Zmiri-strict-provenance $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_strict_provenance $($env:RUSTFLAGS)".Trim() - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Install prerequisites for the `anvil-miri-strict-provenance` recipe. @@ -2674,7 +2674,7 @@ anvil-miri-tree-borrows: anvil-miri-tree-borrows-validate-prereqs anvil-impact $pkg = @(if ($include) { -split $include } else { '--workspace' }) $env:MIRIFLAGS = "-Zmiri-tree-borrows $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_tree_borrows $($env:RUSTFLAGS)".Trim() - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # The three nightly-miri profiles share the same toolchain prereqs @@ -2705,6 +2705,249 @@ anvil-miri-tree-borrows-validate-prereqs: anvil-component-nightly-miri-validate- # scheduled/full tiers (which force ANVIL_IMPACT=off) or a dirty local tree # (which widens for safety). +# Compile all selected packages together, then run their Miri test artifacts in +# parallel. The single Cargo invocation preserves workspace feature unification, +# while artifact-level parallelism uses host logical processors that +# `cargo miri test` otherwise leaves idle. +# +# Packages can opt their own test targets out through: +# +# [package.metadata.anvil.miri] +# exclude = true +# +# They can still compile as dependencies of selected consumers. Workspace runs +# translate the metadata to `--exclude`; impact-scoped runs remove matching +# `--package name@version` pairs before compilation. +# +# ANVIL_MIRI_JOBS overrides the default of one artifact worker per logical processor. + +# Compile selected packages once and run their Miri test artifacts concurrently. +[script("pwsh", "-NoProfile")] +_anvil-miri-test *package_args: + $ErrorActionPreference = 'Stop' + $packageArgs = @('{{ replace(package_args, "'", "''") }}' -split '\s+' | Where-Object { $_ }) + + $metadataJson = & cargo {{_anvil_stable_toolchain_args}} metadata --no-deps --format-version 1 + if ($LASTEXITCODE -ne 0) { + Write-Error 'anvil miri: cargo metadata failed' + exit $LASTEXITCODE + } + try { + $metadata = $metadataJson | ConvertFrom-Json -ErrorAction Stop + } catch { + Write-Error "anvil miri: could not parse cargo metadata output: $($_.Exception.Message)" + exit 1 + } + + $workspaceMembers = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($member in $metadata.workspace_members) { + [void]$workspaceMembers.Add([string]$member) + } + $packageDirectories = @{} + $excludedPackages = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + $excludedPackageIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($package in $metadata.packages) { + $packageId = [string]$package.id + if (-not $workspaceMembers.Contains($packageId)) { continue } + $packageDirectories[$packageId] = Split-Path -Parent ([string]$package.manifest_path) + if ($package.metadata.anvil.miri.exclude -eq $true) { + [void]$excludedPackages.Add([string]$package.name) + [void]$excludedPackageIds.Add($packageId) + } + } + + if ($packageArgs -contains '--workspace') { + if ($workspaceMembers.Count -gt 0 -and $excludedPackageIds.Count -eq $workspaceMembers.Count) { + Write-Host 'anvil miri: all selected packages are excluded; nothing to test' + exit 0 + } + foreach ($packageName in $excludedPackages) { + $packageArgs += @('--exclude', $packageName) + } + } else { + $filtered = [System.Collections.Generic.List[string]]::new() + for ($i = 0; $i -lt $packageArgs.Count; $i++) { + if ($packageArgs[$i] -ne '--package') { + Write-Error "anvil miri: unexpected impact argument '$($packageArgs[$i])'" + exit 1 + } + if (($i + 1) -ge $packageArgs.Count) { + Write-Error 'anvil miri: --package is missing its package spec' + exit 1 + } + $packageSpec = $packageArgs[++$i] + $packageName = ($packageSpec -split '@', 2)[0] + if ($excludedPackages.Contains($packageName)) { continue } + $filtered.Add('--package') + $filtered.Add($packageSpec) + } + $packageArgs = $filtered.ToArray() + if ($packageArgs.Count -eq 0) { + Write-Host 'anvil miri: all selected packages are excluded; nothing to test' + exit 0 + } + } + + $logRoot = Join-Path ([IO.Path]::GetTempPath()) "anvil-miri-$PID-$([guid]::NewGuid())" + New-Item -ItemType Directory -Path $logRoot | Out-Null + try { + $artifactManifest = Join-Path $logRoot 'artifacts.jsonl' + $toolchain = '+{{ rust_nightly }}' + & cargo '+{{ rust_nightly }}' miri test --all-features --tests --no-run --message-format=json-render-diagnostics @packageArgs 1> $artifactManifest + $buildExitCode = $LASTEXITCODE + + $artifacts = [System.Collections.Generic.List[object]]::new() + $seenArtifacts = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($line in Get-Content -LiteralPath $artifactManifest) { + try { + $message = $line | ConvertFrom-Json -ErrorAction Stop + } catch { + Write-Error "anvil miri: could not parse Cargo JSON output: $($_.Exception.Message)" + exit 1 + } + if ($message.reason -eq 'compiler-message' -and $message.message.rendered) { + [Console]::Error.Write($message.message.rendered) + continue + } + if ($message.reason -ne 'compiler-artifact' -or -not $message.executable -or $message.profile.test -ne $true) { + continue + } + $artifactPath = [string]$message.executable + if (-not $seenArtifacts.Add($artifactPath)) { continue } + $packageId = [string]$message.package_id + if ($excludedPackageIds.Contains($packageId)) { continue } + if (-not $packageDirectories.ContainsKey($packageId)) { + Write-Error "anvil miri: package metadata not found for artifact package '$packageId'" + exit 1 + } + $targetName = [string]$message.target.name + $targetKinds = @($message.target.kind | ForEach-Object { [string]$_ }) + if ([string]::IsNullOrWhiteSpace($targetName) -or $targetKinds.Count -eq 0) { + Write-Error "anvil miri: target metadata not found for artifact '$artifactPath'" + exit 1 + } + $targetKind = $targetKinds -join '/' + $artifactFileName = Split-Path -Leaf $artifactPath + $artifacts.Add([pscustomobject]@{ + Path = $artifactPath + PackageId = $packageId + TargetName = $targetName + TargetKind = $targetKind + Label = "$packageId :: $targetKind $targetName ($artifactFileName)" + WorkingDirectory = $packageDirectories[$packageId] + }) + } + + if ($buildExitCode -ne 0) { exit $buildExitCode } + if ($artifacts.Count -eq 0) { + Write-Host 'anvil miri: Cargo produced no runnable test artifacts; nothing to test' + exit 0 + } + + $sysrootOutput = @(& cargo '+{{ rust_nightly }}' miri setup --print-sysroot) + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if ($sysrootOutput.Count -eq 0 -or -not $sysrootOutput[-1]) { + Write-Error 'anvil miri: cargo miri setup did not report a sysroot' + exit 1 + } + $env:MIRI_SYSROOT = $sysrootOutput[-1] + $toolchainSysroot = & rustc $toolchain --print sysroot + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($toolchainSysroot)) { + Write-Error "anvil miri: could not resolve the $toolchain toolchain sysroot" + exit 1 + } + $toolchainBin = Join-Path $toolchainSysroot.Trim() 'bin' + $runner = @( + 'cargo-miri.exe', 'cargo-miri', 'cargo-miri.ps1' | + ForEach-Object { Join-Path $toolchainBin $_ } | + Where-Object { Test-Path -LiteralPath $_ -PathType Leaf } + ) | Select-Object -First 1 + if (-not $runner) { + Write-Error "anvil miri: cargo-miri was not found under '$toolchainBin'" + exit 1 + } + # cargo-miri records MIRI_BE_RUSTC=host in each artifact. Keeping the + # same value in the standalone runner environment lets phase_runner + # suppress that recorded value before launching Miri for interpretation. + $env:MIRI_BE_RUSTC = 'host' + + $jobs = 0 + if ($env:ANVIL_MIRI_JOBS) { + if (-not [int]::TryParse($env:ANVIL_MIRI_JOBS, [ref]$jobs) -or $jobs -lt 1) { + Write-Error "anvil miri: ANVIL_MIRI_JOBS must be a positive integer, got '$($env:ANVIL_MIRI_JOBS)'" + exit 1 + } + } else { + $jobs = [Environment]::ProcessorCount + } + $jobs = [Math]::Min($jobs, $artifacts.Count) + + Write-Host "anvil miri: running $($artifacts.Count) test artifacts with $jobs concurrent process(es)" + + # Cargo normally runs each test executable from its owning package root. + # The direct cargo-miri invocation below restores that working directory. + $orderedArtifacts = @($artifacts | Sort-Object PackageId, TargetKind, TargetName, Path) + $work = for ($i = 0; $i -lt $orderedArtifacts.Count; $i++) { + [pscustomobject]@{ + Index = $i + Artifact = $orderedArtifacts[$i].Path + Label = $orderedArtifacts[$i].Label + WorkingDirectory = $orderedArtifacts[$i].WorkingDirectory + } + } + $results = @( + $work | ForEach-Object -Parallel { + $ErrorActionPreference = 'Stop' + $item = $_ + $log = Join-Path $using:logRoot "$($item.Index).log" + Write-Host "anvil miri: starting $($item.Label)" + Push-Location -LiteralPath $item.WorkingDirectory + try { + & $using:runner runner $item.Artifact *> $log + $exitCode = $LASTEXITCODE + } catch { + $_ | Out-String | Set-Content -LiteralPath $log + $exitCode = 1 + } finally { + Pop-Location + } + Write-Host "anvil miri: completed $($item.Label) (exit $exitCode)" + [pscustomobject]@{ + Index = $item.Index + Label = $item.Label + ExitCode = $exitCode + Log = $log + } + } -ThrottleLimit $jobs + ) + + # Parallel workers buffer into separate logs. Replaying those logs by + # stable artifact index keeps diagnostics isolated and deterministic. + foreach ($result in $results | Sort-Object Index) { + if ($env:GITHUB_ACTIONS -eq 'true') { + Write-Host "::group::Miri artifact $($result.Label)" + } elseif ($env:TF_BUILD -eq 'True') { + Write-Host "##[group]Miri artifact $($result.Label)" + } else { + Write-Host "`n=== Miri artifact $($result.Label) ===" + } + Get-Content -LiteralPath $result.Log | ForEach-Object { Write-Host $_ } + if ($env:GITHUB_ACTIONS -eq 'true') { + Write-Host '::endgroup::' + } elseif ($env:TF_BUILD -eq 'True') { + Write-Host '##[endgroup]' + } + } + + $failed = @($results | Where-Object ExitCode -ne 0) + if ($failed.Count -ne 0) { + [Console]::Error.WriteLine("anvil miri: failed artifacts: $($failed.Label -join ', ')") + exit 1 + } + } finally { + Remove-Item -LiteralPath $logRoot -Recurse -Force -ErrorAction SilentlyContinue + } + # Run Miri tests for affected workspace packages. [script("pwsh", "-NoProfile")] anvil-miri: anvil-miri-validate-prereqs anvil-impact @@ -2736,7 +2979,8 @@ anvil-miri: anvil-miri-validate-prereqs anvil-impact # whose tests are all `#[cfg_attr(miri, ignore)]` -- the canonical # opt-out for build-tooling / CLI crates), so the old nextest # `--no-tests=pass` workaround is no longer needed. - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @(if ($include) { -split $include } else { '--workspace' }) + $pkg = @(if ($include) { -split $include } else { '--workspace' }) + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # --- pr-runtime-analysis members --- @@ -4450,9 +4694,10 @@ anvil-container *command: # The recipe contract's own inputs. These are read by generated checks -- # `anvil-pr-title` reads PR_TITLE, `_anvil-base-ref` reads BASE_REF and its - # CI equivalents, and `anvil-impact` reads ANVIL_IMPACT to decide whether to - # compute scoping, consume a downloaded cache, or skip -- so dropping them at - # the boundary makes the same command mean different things inside and out. + # CI equivalents, `anvil-impact` reads ANVIL_IMPACT to decide whether to + # compute scoping, consume a downloaded cache, or skip, and the Miri runner + # reads ANVIL_MIRI_JOBS to cap artifact workers -- so dropping them at the + # boundary makes the same command mean different things inside and out. # anvil-pr-title is the sharp case: with PR_TITLE unset it exits 0 with a # skip notice, so a title a native run rejects passes in a container and the # tier still reports green. ANVIL_IMPACT is the other: a CI group job exports @@ -4465,7 +4710,7 @@ anvil-container *command: foreach ($name in @( 'PR_TITLE', 'BASE_REF', 'GITHUB_BASE_REF', 'SYSTEM_PULLREQUEST_TARGETBRANCH', - 'ANVIL_IMPACT')) { + 'ANVIL_IMPACT', 'ANVIL_MIRI_JOBS')) { if ((Test-Path -LiteralPath "Env:$name") -and -not [string]::IsNullOrEmpty((Get-Item -LiteralPath "Env:$name").Value)) { $forwardedEnv += $name $runArgs += @('-e', $name) diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap index dd2197ec..3683060b 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap @@ -1486,7 +1486,7 @@ anvil-miri-race-coverage: anvil-miri-race-coverage-validate-prereqs anvil-impact $env:MIRIFLAGS = "-Zmiri-many-seeds=$low..$high $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_race_coverage $($env:RUSTFLAGS)".Trim() Write-Host "anvil-miri-race-coverage: seed window $low..$high (day $day)" - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Install prerequisites for the `anvil-miri-race-coverage` recipe. @@ -1515,7 +1515,7 @@ anvil-miri-strict-provenance: anvil-miri-strict-provenance-validate-prereqs anvi $pkg = @(if ($include) { -split $include } else { '--workspace' }) $env:MIRIFLAGS = "-Zmiri-strict-provenance $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_strict_provenance $($env:RUSTFLAGS)".Trim() - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Install prerequisites for the `anvil-miri-strict-provenance` recipe. @@ -1558,7 +1558,7 @@ anvil-miri-tree-borrows: anvil-miri-tree-borrows-validate-prereqs anvil-impact $pkg = @(if ($include) { -split $include } else { '--workspace' }) $env:MIRIFLAGS = "-Zmiri-tree-borrows $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_tree_borrows $($env:RUSTFLAGS)".Trim() - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # The three nightly-miri profiles share the same toolchain prereqs @@ -1589,6 +1589,249 @@ anvil-miri-tree-borrows-validate-prereqs: anvil-component-nightly-miri-validate- # scheduled/full tiers (which force ANVIL_IMPACT=off) or a dirty local tree # (which widens for safety). +# Compile all selected packages together, then run their Miri test artifacts in +# parallel. The single Cargo invocation preserves workspace feature unification, +# while artifact-level parallelism uses host logical processors that +# `cargo miri test` otherwise leaves idle. +# +# Packages can opt their own test targets out through: +# +# [package.metadata.anvil.miri] +# exclude = true +# +# They can still compile as dependencies of selected consumers. Workspace runs +# translate the metadata to `--exclude`; impact-scoped runs remove matching +# `--package name@version` pairs before compilation. +# +# ANVIL_MIRI_JOBS overrides the default of one artifact worker per logical processor. + +# Compile selected packages once and run their Miri test artifacts concurrently. +[script("pwsh", "-NoProfile")] +_anvil-miri-test *package_args: + $ErrorActionPreference = 'Stop' + $packageArgs = @('{{ replace(package_args, "'", "''") }}' -split '\s+' | Where-Object { $_ }) + + $metadataJson = & cargo {{_anvil_stable_toolchain_args}} metadata --no-deps --format-version 1 + if ($LASTEXITCODE -ne 0) { + Write-Error 'anvil miri: cargo metadata failed' + exit $LASTEXITCODE + } + try { + $metadata = $metadataJson | ConvertFrom-Json -ErrorAction Stop + } catch { + Write-Error "anvil miri: could not parse cargo metadata output: $($_.Exception.Message)" + exit 1 + } + + $workspaceMembers = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($member in $metadata.workspace_members) { + [void]$workspaceMembers.Add([string]$member) + } + $packageDirectories = @{} + $excludedPackages = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + $excludedPackageIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($package in $metadata.packages) { + $packageId = [string]$package.id + if (-not $workspaceMembers.Contains($packageId)) { continue } + $packageDirectories[$packageId] = Split-Path -Parent ([string]$package.manifest_path) + if ($package.metadata.anvil.miri.exclude -eq $true) { + [void]$excludedPackages.Add([string]$package.name) + [void]$excludedPackageIds.Add($packageId) + } + } + + if ($packageArgs -contains '--workspace') { + if ($workspaceMembers.Count -gt 0 -and $excludedPackageIds.Count -eq $workspaceMembers.Count) { + Write-Host 'anvil miri: all selected packages are excluded; nothing to test' + exit 0 + } + foreach ($packageName in $excludedPackages) { + $packageArgs += @('--exclude', $packageName) + } + } else { + $filtered = [System.Collections.Generic.List[string]]::new() + for ($i = 0; $i -lt $packageArgs.Count; $i++) { + if ($packageArgs[$i] -ne '--package') { + Write-Error "anvil miri: unexpected impact argument '$($packageArgs[$i])'" + exit 1 + } + if (($i + 1) -ge $packageArgs.Count) { + Write-Error 'anvil miri: --package is missing its package spec' + exit 1 + } + $packageSpec = $packageArgs[++$i] + $packageName = ($packageSpec -split '@', 2)[0] + if ($excludedPackages.Contains($packageName)) { continue } + $filtered.Add('--package') + $filtered.Add($packageSpec) + } + $packageArgs = $filtered.ToArray() + if ($packageArgs.Count -eq 0) { + Write-Host 'anvil miri: all selected packages are excluded; nothing to test' + exit 0 + } + } + + $logRoot = Join-Path ([IO.Path]::GetTempPath()) "anvil-miri-$PID-$([guid]::NewGuid())" + New-Item -ItemType Directory -Path $logRoot | Out-Null + try { + $artifactManifest = Join-Path $logRoot 'artifacts.jsonl' + $toolchain = '+{{ rust_nightly }}' + & cargo '+{{ rust_nightly }}' miri test --all-features --tests --no-run --message-format=json-render-diagnostics @packageArgs 1> $artifactManifest + $buildExitCode = $LASTEXITCODE + + $artifacts = [System.Collections.Generic.List[object]]::new() + $seenArtifacts = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($line in Get-Content -LiteralPath $artifactManifest) { + try { + $message = $line | ConvertFrom-Json -ErrorAction Stop + } catch { + Write-Error "anvil miri: could not parse Cargo JSON output: $($_.Exception.Message)" + exit 1 + } + if ($message.reason -eq 'compiler-message' -and $message.message.rendered) { + [Console]::Error.Write($message.message.rendered) + continue + } + if ($message.reason -ne 'compiler-artifact' -or -not $message.executable -or $message.profile.test -ne $true) { + continue + } + $artifactPath = [string]$message.executable + if (-not $seenArtifacts.Add($artifactPath)) { continue } + $packageId = [string]$message.package_id + if ($excludedPackageIds.Contains($packageId)) { continue } + if (-not $packageDirectories.ContainsKey($packageId)) { + Write-Error "anvil miri: package metadata not found for artifact package '$packageId'" + exit 1 + } + $targetName = [string]$message.target.name + $targetKinds = @($message.target.kind | ForEach-Object { [string]$_ }) + if ([string]::IsNullOrWhiteSpace($targetName) -or $targetKinds.Count -eq 0) { + Write-Error "anvil miri: target metadata not found for artifact '$artifactPath'" + exit 1 + } + $targetKind = $targetKinds -join '/' + $artifactFileName = Split-Path -Leaf $artifactPath + $artifacts.Add([pscustomobject]@{ + Path = $artifactPath + PackageId = $packageId + TargetName = $targetName + TargetKind = $targetKind + Label = "$packageId :: $targetKind $targetName ($artifactFileName)" + WorkingDirectory = $packageDirectories[$packageId] + }) + } + + if ($buildExitCode -ne 0) { exit $buildExitCode } + if ($artifacts.Count -eq 0) { + Write-Host 'anvil miri: Cargo produced no runnable test artifacts; nothing to test' + exit 0 + } + + $sysrootOutput = @(& cargo '+{{ rust_nightly }}' miri setup --print-sysroot) + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if ($sysrootOutput.Count -eq 0 -or -not $sysrootOutput[-1]) { + Write-Error 'anvil miri: cargo miri setup did not report a sysroot' + exit 1 + } + $env:MIRI_SYSROOT = $sysrootOutput[-1] + $toolchainSysroot = & rustc $toolchain --print sysroot + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($toolchainSysroot)) { + Write-Error "anvil miri: could not resolve the $toolchain toolchain sysroot" + exit 1 + } + $toolchainBin = Join-Path $toolchainSysroot.Trim() 'bin' + $runner = @( + 'cargo-miri.exe', 'cargo-miri', 'cargo-miri.ps1' | + ForEach-Object { Join-Path $toolchainBin $_ } | + Where-Object { Test-Path -LiteralPath $_ -PathType Leaf } + ) | Select-Object -First 1 + if (-not $runner) { + Write-Error "anvil miri: cargo-miri was not found under '$toolchainBin'" + exit 1 + } + # cargo-miri records MIRI_BE_RUSTC=host in each artifact. Keeping the + # same value in the standalone runner environment lets phase_runner + # suppress that recorded value before launching Miri for interpretation. + $env:MIRI_BE_RUSTC = 'host' + + $jobs = 0 + if ($env:ANVIL_MIRI_JOBS) { + if (-not [int]::TryParse($env:ANVIL_MIRI_JOBS, [ref]$jobs) -or $jobs -lt 1) { + Write-Error "anvil miri: ANVIL_MIRI_JOBS must be a positive integer, got '$($env:ANVIL_MIRI_JOBS)'" + exit 1 + } + } else { + $jobs = [Environment]::ProcessorCount + } + $jobs = [Math]::Min($jobs, $artifacts.Count) + + Write-Host "anvil miri: running $($artifacts.Count) test artifacts with $jobs concurrent process(es)" + + # Cargo normally runs each test executable from its owning package root. + # The direct cargo-miri invocation below restores that working directory. + $orderedArtifacts = @($artifacts | Sort-Object PackageId, TargetKind, TargetName, Path) + $work = for ($i = 0; $i -lt $orderedArtifacts.Count; $i++) { + [pscustomobject]@{ + Index = $i + Artifact = $orderedArtifacts[$i].Path + Label = $orderedArtifacts[$i].Label + WorkingDirectory = $orderedArtifacts[$i].WorkingDirectory + } + } + $results = @( + $work | ForEach-Object -Parallel { + $ErrorActionPreference = 'Stop' + $item = $_ + $log = Join-Path $using:logRoot "$($item.Index).log" + Write-Host "anvil miri: starting $($item.Label)" + Push-Location -LiteralPath $item.WorkingDirectory + try { + & $using:runner runner $item.Artifact *> $log + $exitCode = $LASTEXITCODE + } catch { + $_ | Out-String | Set-Content -LiteralPath $log + $exitCode = 1 + } finally { + Pop-Location + } + Write-Host "anvil miri: completed $($item.Label) (exit $exitCode)" + [pscustomobject]@{ + Index = $item.Index + Label = $item.Label + ExitCode = $exitCode + Log = $log + } + } -ThrottleLimit $jobs + ) + + # Parallel workers buffer into separate logs. Replaying those logs by + # stable artifact index keeps diagnostics isolated and deterministic. + foreach ($result in $results | Sort-Object Index) { + if ($env:GITHUB_ACTIONS -eq 'true') { + Write-Host "::group::Miri artifact $($result.Label)" + } elseif ($env:TF_BUILD -eq 'True') { + Write-Host "##[group]Miri artifact $($result.Label)" + } else { + Write-Host "`n=== Miri artifact $($result.Label) ===" + } + Get-Content -LiteralPath $result.Log | ForEach-Object { Write-Host $_ } + if ($env:GITHUB_ACTIONS -eq 'true') { + Write-Host '::endgroup::' + } elseif ($env:TF_BUILD -eq 'True') { + Write-Host '##[endgroup]' + } + } + + $failed = @($results | Where-Object ExitCode -ne 0) + if ($failed.Count -ne 0) { + [Console]::Error.WriteLine("anvil miri: failed artifacts: $($failed.Label -join ', ')") + exit 1 + } + } finally { + Remove-Item -LiteralPath $logRoot -Recurse -Force -ErrorAction SilentlyContinue + } + # Run Miri tests for affected workspace packages. [script("pwsh", "-NoProfile")] anvil-miri: anvil-miri-validate-prereqs anvil-impact @@ -1620,7 +1863,8 @@ anvil-miri: anvil-miri-validate-prereqs anvil-impact # whose tests are all `#[cfg_attr(miri, ignore)]` -- the canonical # opt-out for build-tooling / CLI crates), so the old nextest # `--no-tests=pass` workaround is no longer needed. - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @(if ($include) { -split $include } else { '--workspace' }) + $pkg = @(if ($include) { -split $include } else { '--workspace' }) + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # --- pr-runtime-analysis members --- @@ -3334,9 +3578,10 @@ anvil-container *command: # The recipe contract's own inputs. These are read by generated checks -- # `anvil-pr-title` reads PR_TITLE, `_anvil-base-ref` reads BASE_REF and its - # CI equivalents, and `anvil-impact` reads ANVIL_IMPACT to decide whether to - # compute scoping, consume a downloaded cache, or skip -- so dropping them at - # the boundary makes the same command mean different things inside and out. + # CI equivalents, `anvil-impact` reads ANVIL_IMPACT to decide whether to + # compute scoping, consume a downloaded cache, or skip, and the Miri runner + # reads ANVIL_MIRI_JOBS to cap artifact workers -- so dropping them at the + # boundary makes the same command mean different things inside and out. # anvil-pr-title is the sharp case: with PR_TITLE unset it exits 0 with a # skip notice, so a title a native run rejects passes in a container and the # tier still reports green. ANVIL_IMPACT is the other: a CI group job exports @@ -3349,7 +3594,7 @@ anvil-container *command: foreach ($name in @( 'PR_TITLE', 'BASE_REF', 'GITHUB_BASE_REF', 'SYSTEM_PULLREQUEST_TARGETBRANCH', - 'ANVIL_IMPACT')) { + 'ANVIL_IMPACT', 'ANVIL_MIRI_JOBS')) { if ((Test-Path -LiteralPath "Env:$name") -and -not [string]::IsNullOrEmpty((Get-Item -LiteralPath "Env:$name").Value)) { $forwardedEnv += $name $runArgs += @('-e', $name) diff --git a/justfiles/anvil/checks/miri-race-coverage.just b/justfiles/anvil/checks/miri-race-coverage.just index 4851799b..de4a2716 100644 --- a/justfiles/anvil/checks/miri-race-coverage.just +++ b/justfiles/anvil/checks/miri-race-coverage.just @@ -26,7 +26,7 @@ anvil-miri-race-coverage: anvil-miri-race-coverage-validate-prereqs anvil-impact $env:MIRIFLAGS = "-Zmiri-many-seeds=$low..$high $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_race_coverage $($env:RUSTFLAGS)".Trim() Write-Host "anvil-miri-race-coverage: seed window $low..$high (day $day)" - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Install prerequisites for the `anvil-miri-race-coverage` recipe. diff --git a/justfiles/anvil/checks/miri-strict-provenance.just b/justfiles/anvil/checks/miri-strict-provenance.just index 60f36c27..65de202b 100644 --- a/justfiles/anvil/checks/miri-strict-provenance.just +++ b/justfiles/anvil/checks/miri-strict-provenance.just @@ -15,7 +15,7 @@ anvil-miri-strict-provenance: anvil-miri-strict-provenance-validate-prereqs anvi $pkg = @(if ($include) { -split $include } else { '--workspace' }) $env:MIRIFLAGS = "-Zmiri-strict-provenance $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_strict_provenance $($env:RUSTFLAGS)".Trim() - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # Install prerequisites for the `anvil-miri-strict-provenance` recipe. diff --git a/justfiles/anvil/checks/miri-tree-borrows.just b/justfiles/anvil/checks/miri-tree-borrows.just index c1786354..eccdcd8f 100644 --- a/justfiles/anvil/checks/miri-tree-borrows.just +++ b/justfiles/anvil/checks/miri-tree-borrows.just @@ -29,7 +29,7 @@ anvil-miri-tree-borrows: anvil-miri-tree-borrows-validate-prereqs anvil-impact $pkg = @(if ($include) { -split $include } else { '--workspace' }) $env:MIRIFLAGS = "-Zmiri-tree-borrows $($env:MIRIFLAGS)".Trim() $env:RUSTFLAGS = "--cfg miri_tree_borrows $($env:RUSTFLAGS)".Trim() - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @pkg + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # The three nightly-miri profiles share the same toolchain prereqs diff --git a/justfiles/anvil/checks/miri.just b/justfiles/anvil/checks/miri.just index 630e2c79..10f5ca6b 100644 --- a/justfiles/anvil/checks/miri.just +++ b/justfiles/anvil/checks/miri.just @@ -12,6 +12,249 @@ # scheduled/full tiers (which force ANVIL_IMPACT=off) or a dirty local tree # (which widens for safety). +# Compile all selected packages together, then run their Miri test artifacts in +# parallel. The single Cargo invocation preserves workspace feature unification, +# while artifact-level parallelism uses host logical processors that +# `cargo miri test` otherwise leaves idle. +# +# Packages can opt their own test targets out through: +# +# [package.metadata.anvil.miri] +# exclude = true +# +# They can still compile as dependencies of selected consumers. Workspace runs +# translate the metadata to `--exclude`; impact-scoped runs remove matching +# `--package name@version` pairs before compilation. +# +# ANVIL_MIRI_JOBS overrides the default of one artifact worker per logical processor. + +# Compile selected packages once and run their Miri test artifacts concurrently. +[script("pwsh", "-NoProfile")] +_anvil-miri-test *package_args: + $ErrorActionPreference = 'Stop' + $packageArgs = @('{{ replace(package_args, "'", "''") }}' -split '\s+' | Where-Object { $_ }) + + $metadataJson = & cargo {{_anvil_stable_toolchain_args}} metadata --no-deps --format-version 1 + if ($LASTEXITCODE -ne 0) { + Write-Error 'anvil miri: cargo metadata failed' + exit $LASTEXITCODE + } + try { + $metadata = $metadataJson | ConvertFrom-Json -ErrorAction Stop + } catch { + Write-Error "anvil miri: could not parse cargo metadata output: $($_.Exception.Message)" + exit 1 + } + + $workspaceMembers = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($member in $metadata.workspace_members) { + [void]$workspaceMembers.Add([string]$member) + } + $packageDirectories = @{} + $excludedPackages = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + $excludedPackageIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($package in $metadata.packages) { + $packageId = [string]$package.id + if (-not $workspaceMembers.Contains($packageId)) { continue } + $packageDirectories[$packageId] = Split-Path -Parent ([string]$package.manifest_path) + if ($package.metadata.anvil.miri.exclude -eq $true) { + [void]$excludedPackages.Add([string]$package.name) + [void]$excludedPackageIds.Add($packageId) + } + } + + if ($packageArgs -contains '--workspace') { + if ($workspaceMembers.Count -gt 0 -and $excludedPackageIds.Count -eq $workspaceMembers.Count) { + Write-Host 'anvil miri: all selected packages are excluded; nothing to test' + exit 0 + } + foreach ($packageName in $excludedPackages) { + $packageArgs += @('--exclude', $packageName) + } + } else { + $filtered = [System.Collections.Generic.List[string]]::new() + for ($i = 0; $i -lt $packageArgs.Count; $i++) { + if ($packageArgs[$i] -ne '--package') { + Write-Error "anvil miri: unexpected impact argument '$($packageArgs[$i])'" + exit 1 + } + if (($i + 1) -ge $packageArgs.Count) { + Write-Error 'anvil miri: --package is missing its package spec' + exit 1 + } + $packageSpec = $packageArgs[++$i] + $packageName = ($packageSpec -split '@', 2)[0] + if ($excludedPackages.Contains($packageName)) { continue } + $filtered.Add('--package') + $filtered.Add($packageSpec) + } + $packageArgs = $filtered.ToArray() + if ($packageArgs.Count -eq 0) { + Write-Host 'anvil miri: all selected packages are excluded; nothing to test' + exit 0 + } + } + + $logRoot = Join-Path ([IO.Path]::GetTempPath()) "anvil-miri-$PID-$([guid]::NewGuid())" + New-Item -ItemType Directory -Path $logRoot | Out-Null + try { + $artifactManifest = Join-Path $logRoot 'artifacts.jsonl' + $toolchain = '+{{ rust_nightly }}' + & cargo '+{{ rust_nightly }}' miri test --all-features --tests --no-run --message-format=json-render-diagnostics @packageArgs 1> $artifactManifest + $buildExitCode = $LASTEXITCODE + + $artifacts = [System.Collections.Generic.List[object]]::new() + $seenArtifacts = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal) + foreach ($line in Get-Content -LiteralPath $artifactManifest) { + try { + $message = $line | ConvertFrom-Json -ErrorAction Stop + } catch { + Write-Error "anvil miri: could not parse Cargo JSON output: $($_.Exception.Message)" + exit 1 + } + if ($message.reason -eq 'compiler-message' -and $message.message.rendered) { + [Console]::Error.Write($message.message.rendered) + continue + } + if ($message.reason -ne 'compiler-artifact' -or -not $message.executable -or $message.profile.test -ne $true) { + continue + } + $artifactPath = [string]$message.executable + if (-not $seenArtifacts.Add($artifactPath)) { continue } + $packageId = [string]$message.package_id + if ($excludedPackageIds.Contains($packageId)) { continue } + if (-not $packageDirectories.ContainsKey($packageId)) { + Write-Error "anvil miri: package metadata not found for artifact package '$packageId'" + exit 1 + } + $targetName = [string]$message.target.name + $targetKinds = @($message.target.kind | ForEach-Object { [string]$_ }) + if ([string]::IsNullOrWhiteSpace($targetName) -or $targetKinds.Count -eq 0) { + Write-Error "anvil miri: target metadata not found for artifact '$artifactPath'" + exit 1 + } + $targetKind = $targetKinds -join '/' + $artifactFileName = Split-Path -Leaf $artifactPath + $artifacts.Add([pscustomobject]@{ + Path = $artifactPath + PackageId = $packageId + TargetName = $targetName + TargetKind = $targetKind + Label = "$packageId :: $targetKind $targetName ($artifactFileName)" + WorkingDirectory = $packageDirectories[$packageId] + }) + } + + if ($buildExitCode -ne 0) { exit $buildExitCode } + if ($artifacts.Count -eq 0) { + Write-Host 'anvil miri: Cargo produced no runnable test artifacts; nothing to test' + exit 0 + } + + $sysrootOutput = @(& cargo '+{{ rust_nightly }}' miri setup --print-sysroot) + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if ($sysrootOutput.Count -eq 0 -or -not $sysrootOutput[-1]) { + Write-Error 'anvil miri: cargo miri setup did not report a sysroot' + exit 1 + } + $env:MIRI_SYSROOT = $sysrootOutput[-1] + $toolchainSysroot = & rustc $toolchain --print sysroot + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($toolchainSysroot)) { + Write-Error "anvil miri: could not resolve the $toolchain toolchain sysroot" + exit 1 + } + $toolchainBin = Join-Path $toolchainSysroot.Trim() 'bin' + $runner = @( + 'cargo-miri.exe', 'cargo-miri', 'cargo-miri.ps1' | + ForEach-Object { Join-Path $toolchainBin $_ } | + Where-Object { Test-Path -LiteralPath $_ -PathType Leaf } + ) | Select-Object -First 1 + if (-not $runner) { + Write-Error "anvil miri: cargo-miri was not found under '$toolchainBin'" + exit 1 + } + # cargo-miri records MIRI_BE_RUSTC=host in each artifact. Keeping the + # same value in the standalone runner environment lets phase_runner + # suppress that recorded value before launching Miri for interpretation. + $env:MIRI_BE_RUSTC = 'host' + + $jobs = 0 + if ($env:ANVIL_MIRI_JOBS) { + if (-not [int]::TryParse($env:ANVIL_MIRI_JOBS, [ref]$jobs) -or $jobs -lt 1) { + Write-Error "anvil miri: ANVIL_MIRI_JOBS must be a positive integer, got '$($env:ANVIL_MIRI_JOBS)'" + exit 1 + } + } else { + $jobs = [Environment]::ProcessorCount + } + $jobs = [Math]::Min($jobs, $artifacts.Count) + + Write-Host "anvil miri: running $($artifacts.Count) test artifacts with $jobs concurrent process(es)" + + # Cargo normally runs each test executable from its owning package root. + # The direct cargo-miri invocation below restores that working directory. + $orderedArtifacts = @($artifacts | Sort-Object PackageId, TargetKind, TargetName, Path) + $work = for ($i = 0; $i -lt $orderedArtifacts.Count; $i++) { + [pscustomobject]@{ + Index = $i + Artifact = $orderedArtifacts[$i].Path + Label = $orderedArtifacts[$i].Label + WorkingDirectory = $orderedArtifacts[$i].WorkingDirectory + } + } + $results = @( + $work | ForEach-Object -Parallel { + $ErrorActionPreference = 'Stop' + $item = $_ + $log = Join-Path $using:logRoot "$($item.Index).log" + Write-Host "anvil miri: starting $($item.Label)" + Push-Location -LiteralPath $item.WorkingDirectory + try { + & $using:runner runner $item.Artifact *> $log + $exitCode = $LASTEXITCODE + } catch { + $_ | Out-String | Set-Content -LiteralPath $log + $exitCode = 1 + } finally { + Pop-Location + } + Write-Host "anvil miri: completed $($item.Label) (exit $exitCode)" + [pscustomobject]@{ + Index = $item.Index + Label = $item.Label + ExitCode = $exitCode + Log = $log + } + } -ThrottleLimit $jobs + ) + + # Parallel workers buffer into separate logs. Replaying those logs by + # stable artifact index keeps diagnostics isolated and deterministic. + foreach ($result in $results | Sort-Object Index) { + if ($env:GITHUB_ACTIONS -eq 'true') { + Write-Host "::group::Miri artifact $($result.Label)" + } elseif ($env:TF_BUILD -eq 'True') { + Write-Host "##[group]Miri artifact $($result.Label)" + } else { + Write-Host "`n=== Miri artifact $($result.Label) ===" + } + Get-Content -LiteralPath $result.Log | ForEach-Object { Write-Host $_ } + if ($env:GITHUB_ACTIONS -eq 'true') { + Write-Host '::endgroup::' + } elseif ($env:TF_BUILD -eq 'True') { + Write-Host '##[endgroup]' + } + } + + $failed = @($results | Where-Object ExitCode -ne 0) + if ($failed.Count -ne 0) { + [Console]::Error.WriteLine("anvil miri: failed artifacts: $($failed.Label -join ', ')") + exit 1 + } + } finally { + Remove-Item -LiteralPath $logRoot -Recurse -Force -ErrorAction SilentlyContinue + } + # Run Miri tests for affected workspace packages. [script("pwsh", "-NoProfile")] anvil-miri: anvil-miri-validate-prereqs anvil-impact @@ -43,7 +286,8 @@ anvil-miri: anvil-miri-validate-prereqs anvil-impact # whose tests are all `#[cfg_attr(miri, ignore)]` -- the canonical # opt-out for build-tooling / CLI crates), so the old nextest # `--no-tests=pass` workaround is no longer needed. - & cargo '+{{ rust_nightly }}' miri test --all-features --tests @(if ($include) { -split $include } else { '--workspace' }) + $pkg = @(if ($include) { -split $include } else { '--workspace' }) + & "{{ just_executable() }}" _anvil-miri-test @pkg if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # --- pr-runtime-analysis members --- diff --git a/justfiles/anvil/container.just b/justfiles/anvil/container.just index 6e3a91d8..82a98d51 100644 --- a/justfiles/anvil/container.just +++ b/justfiles/anvil/container.just @@ -972,9 +972,10 @@ anvil-container *command: # The recipe contract's own inputs. These are read by generated checks -- # `anvil-pr-title` reads PR_TITLE, `_anvil-base-ref` reads BASE_REF and its - # CI equivalents, and `anvil-impact` reads ANVIL_IMPACT to decide whether to - # compute scoping, consume a downloaded cache, or skip -- so dropping them at - # the boundary makes the same command mean different things inside and out. + # CI equivalents, `anvil-impact` reads ANVIL_IMPACT to decide whether to + # compute scoping, consume a downloaded cache, or skip, and the Miri runner + # reads ANVIL_MIRI_JOBS to cap artifact workers -- so dropping them at the + # boundary makes the same command mean different things inside and out. # anvil-pr-title is the sharp case: with PR_TITLE unset it exits 0 with a # skip notice, so a title a native run rejects passes in a container and the # tier still reports green. ANVIL_IMPACT is the other: a CI group job exports @@ -987,7 +988,7 @@ anvil-container *command: foreach ($name in @( 'PR_TITLE', 'BASE_REF', 'GITHUB_BASE_REF', 'SYSTEM_PULLREQUEST_TARGETBRANCH', - 'ANVIL_IMPACT')) { + 'ANVIL_IMPACT', 'ANVIL_MIRI_JOBS')) { if ((Test-Path -LiteralPath "Env:$name") -and -not [string]::IsNullOrEmpty((Get-Item -LiteralPath "Env:$name").Value)) { $forwardedEnv += $name $runArgs += @('-e', $name)