Run tests with nextest and publish results

Author: bantonsson

.config/nextest.toml

@@ -0,0 +1,36 @@
+[store]
+dir = "target/nextest"
+
+[test-groups]
+single-threaded = { max-threads = 1 }
+
+[profile.default]
+# Show tests with status skip/pass/slow/fail while running.
+status-level = "skip"
+# Only show tests with status slow/fail at the end of the run.
+final-status-level = "slow"
+# Print out output for failing tests as soon as they fail.
+failure-output = "immediate"
+# Stop the test run on the first failure.
+fail-fast = true
+
+[[profile.default.overrides]]
+filter = 'test(::single_threaded_tests::)'
+test-group = 'single-threaded'
+
+[profile.ci]
+# Print out output for failing tests as soon as they fail, and also at the end
+# of the run (for easy scrollability).
+failure-output = "immediate-final"
+# Do not cancel the test run on the first failure.
+fail-fast = false
+
+[profile.ci.junit]
+# Output a JUnit report into the given file inside 'store.dir/<profile-name>'.
+path = "junit.xml"
+# The name of the top-level "report" element in JUnit report.
+report-name = "nextest-run"
+# Don't include stdout and stderr in the JUnit report for successes.
+store-success-output = false
+# Include the stdout and stderr in the JUnit report for failures.
+store-failure-output = true

.github/workflows/coverage.yml

@@ -14,13 +14,16 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Install Rust
-        run: rustup update stable
+        run: rustup install nightly && rustup default nightly
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@v2
         with:
-          tool: [email protected]
-      - name: Generate code coverage
-        run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info
+          tool: [email protected],[email protected]
+      - name: Generate code coverage (including doc tests)
+        run: |
+          cargo llvm-cov --all-features --workspace --no-report nextest
+          cargo llvm-cov --all-features --workspace --no-report --doc
+          cargo llvm-cov report --doctests --lcov --output-path lcov.info
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

.github/workflows/miri.yml

@@ -5,6 +5,8 @@ on:
 jobs:
   run-miri:
     runs-on: ubuntu-latest
+    env:
+      CARGO_TERM_COLOR: always
     steps:
     - uses: actions/checkout@v2
     - uses: actions-rs/toolchain@v1
@@ -13,6 +15,9 @@ jobs:
         toolchain: nightly
         override: true
         components: miri
-    - run: MIRIFLAGS="-Zmiri-disable-isolation" cargo miri test
+    - uses: taiki-e/install-action@v2
+      with: 
+        tool: [email protected]
+    - run: MIRIFLAGS="-Zmiri-disable-isolation" cargo miri nextest run
     # We need to disable isolation because 
     # "unsupported operation: `clock_gettime` with `REALTIME` clocks not available when isolation is enabled" 

.github/workflows/test.yml

@@ -29,6 +29,13 @@ jobs:
       - id: rust-version
         # On Windows run happens in a PowerShell, so start bash explicitly
         run: bash -c 'echo "version=$(rustc --version)" >> $GITHUB_OUTPUT'
+      - name: Install cargo nextest
+        uses: taiki-e/install-action@v2
+        with: 
+          tool: [email protected]
+      - name: "Remove nextest CI report"
+        shell: bash
+        run: rm -rf target/nextest/ci/junit.xml
       - name: Free Disk Space (Ubuntu only)
         if: runner.os == 'Linux' && matrix.platform == 'ubuntu-latest'
         uses: jlumbroso/[email protected]
@@ -42,19 +49,23 @@ jobs:
           swap-storage: true
       - name: "[${{ steps.rust-version.outputs.version}}] cargo build --workspace --verbose"
         run: cargo build --workspace --verbose
-      - name: "[${{ steps.rust-version.outputs.version}}] cargo test --workspace --verbose"
-        run: cargo test --workspace --verbose
-        env:
-          RUST_BACKTRACE: 1
-      - name: "[${{ steps.rust-version.outputs.version}}] cargo test --workspace --verbose -- --ignored --test-threads=1"
-        run: cargo test --workspace --verbose -- --ignored --test-threads=1
+      - name: "[${{ steps.rust-version.outputs.version}}] cargo nextest run --workspace --profile ci --verbose"
+        # Run doc tests with cargo test and run tests with nextest and generate junit.xml
+        run: cargo test --workspace --doc --verbose && cargo nextest run --workspace --profile ci --verbose
         env:
           RUST_BACKTRACE: 1
-      - name: "[${{ steps.rust-version.outputs.version}}] RUSTFLAGS=\"-C prefer-dynamic\" cargo test --package test_spawn_from_lib --features prefer-dynamic -- --ignored"
-        run: cargo test --package test_spawn_from_lib --features prefer-dynamic -- --ignored
+      - name: "[${{ steps.rust-version.outputs.version}}] RUSTFLAGS=\"-C prefer-dynamic\" cargo nextest run --package test_spawn_from_lib --features prefer-dynamic"
+        run: cargo nextest run --package test_spawn_from_lib --features prefer-dynamic
         env:
           RUSTFLAGS: "-C prefer-dynamic"
           RUST_BACKTRACE: 1
+      - name: Report Test Results
+        if: success() || failure() 
+        uses: mikepenz/action-junit-report@v4
+        with:
+          report_paths: "target/nextest/ci/junit.xml"
+          check_name: "[${{ matrix.platform }}:${{ matrix.rust_version }}] test report"
+          include_passed: true
 
   ffi:
     name: "FFI #${{ matrix.platform }} ${{ matrix.rust_version }}"
@@ -94,7 +105,8 @@ jobs:
         run: rustup install ${{ matrix.rust_version }} && rustup default ${{ matrix.rust_version }}
 
       - id: rust-version
-        run: echo "version=$(rustc --version)" >> $GITHUB_OUTPUT
+        # On Windows run happens in a PowerShell, so start bash explicitly
+        run: bash -c 'echo "version=$(rustc --version)" >> $GITHUB_OUTPUT'
 
       - name: "Generate profiling FFI"
         shell: bash
@@ -168,8 +180,9 @@ jobs:
         with:
           rust_version: cross-centos7
       - run: cargo install cross || true
-      - run: cross build --all
-      - run: cross test --all
+      - run: cross build --workspace --target x86_64-unknown-linux-gnu
+      - run: cross test --workspace --target x86_64-unknown-linux-gnu -- --skip "::single_threaded_tests::"
+      - run: cross test --workspace --target x86_64-unknown-linux-gnu --exclude bin_tests -- --skip "::tests::" --skip "::api_tests::" --test-threads 1
 
   ffi_bake:
     strategy:

README.md

@@ -31,3 +31,23 @@ bash build-profiling-ffi.sh /opt/libdatadog
 
 - Rust 1.71 or newer with cargo
 - `cmake` and `protoc`
+
+### Running tests
+
+This project uses [cargo-nextest][nt] to run tests.
+
+```bash
+cargo nextest run
+```
+
+#### Installing cargo-nextest
+
+The simplest way to install [cargo-nextest][nt] is to use `cargo install` like this.
+
+```bash
+cargo install --locked '[email protected]'
+```
+
+Please note that the locked version is to make sure that it can be built using rust `1.71.1`, and if you are using a newer rust version, then it's enough to limit the version to `0.9.*`.
+
+[nt]: https://nexte.st/

crashtracker/src/api.rs

@@ -10,11 +10,8 @@ use crate::{
         update_receiver_after_fork,
     },
     crash_info::CrashtrackerMetadata,
-    update_config, update_metadata, CrashtrackerConfiguration, StacktraceCollection,
+    update_config, update_metadata, CrashtrackerConfiguration,
 };
-use ddcommon::tag::Tag;
-use ddcommon::Endpoint;
-use std::time::Duration;
 
 /// Cleans up after the crash-tracker:
 /// Unregister the crash handler, restore the previous handler (if any), and
@@ -101,22 +98,21 @@ pub fn init(
     Ok(())
 }
 
-#[ignore]
-#[allow(dead_code)]
-// Ignored tests are still run in CI.
-// To test, uncomment the line below than run manually
 // We can't run this in the main test runner because it (deliberately) crashes,
 // and would make all following tests unrunable.
 // To run this test,
 // ./build-profiling-ffi /tmp/libdatadog
 // mkdir /tmp/crashreports
 // look in /tmp/crashreports for the crash reports and output files
-// Commented out since `ignore` doesn't work in CI.
-//#[test]
+#[ignore]
+#[test]
 fn test_crash() {
-    use crate::begin_profiling_op;
+    use crate::{begin_profiling_op, StacktraceCollection};
     use chrono::Utc;
     use ddcommon::parse_uri;
+    use ddcommon::tag::Tag;
+    use ddcommon::Endpoint;
+    use std::time::Duration;
 
     let time = Utc::now().to_rfc3339();
     let dir = "/tmp/crashreports/";

ddcommon-ffi/src/slice.rs

@@ -143,7 +143,7 @@ impl<'a> From<&'a str> for Slice<'a, c_char> {
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use std::os::raw::c_char;
 
     use crate::slice::*;

ddcommon-ffi/src/vec.rs

@@ -114,7 +114,7 @@ impl<T> Default for Vec<T> {
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use super::*;
 
     #[test]

ddtelemetry-ffi/src/lib.rs

@@ -105,7 +105,7 @@ macro_rules! try_c {
 pub(crate) use c_setters;
 
 #[cfg(test)]
-mod test_c_ffi {
+mod tests {
     use crate::{builder::*, worker_handle::*};
     use ddcommon::{parse_uri, Endpoint};
     use ddcommon_ffi as ffi;

ddtelemetry/src/config.rs

@@ -213,7 +213,7 @@ impl Config {
 }
 
 #[cfg(all(test, target_family = "unix"))]
-mod test {
+mod tests {
     use ddcommon::connector::uds;
 
     use super::Config;

ddtelemetry/src/metrics.rs

@@ -219,7 +219,7 @@ impl MetricContexts {
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use std::fmt::Debug;
 
     use super::*;

ipc/src/platform/unix/mod.rs

@@ -24,7 +24,7 @@ pub unsafe extern "C" fn memfd_create(name: libc::c_void, flags: libc::c_uint) -
 }
 
 #[cfg(test)]
-mod tests {
+mod single_threaded_tests {
     use io_lifetimes::OwnedFd;
     use pretty_assertions::assert_eq;
     use std::{
@@ -90,8 +90,8 @@ mod tests {
         assert_eq!(reference_meta, &current_meta);
     }
 
+    // tests checks global FD state - so it needs to run single-threaded
     #[test]
-    #[ignore] // tests checks global FD state - so it needs to run in non-parallel mode
     fn test_channel_metadata_only_provides_valid_owned() {
         let reference = get_open_file_descriptors(None).unwrap();
         let mut meta = ChannelMetadata::default();

profiling-ffi/src/exporter.rs

@@ -459,7 +459,7 @@ pub unsafe extern "C" fn ddog_CancellationToken_drop(token: Option<&mut Cancella
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use super::*;
     use ddcommon_ffi::Slice;
     use serde_json::json;

profiling-ffi/src/profiles.rs

@@ -742,7 +742,7 @@ pub unsafe extern "C" fn ddog_prof_Profile_reset(
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use super::*;
 
     #[test]

profiling/src/api.rs

@@ -344,7 +344,7 @@ impl<'a> TryFrom<&'a pprof::Profile> for Profile<'a> {
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use super::*;
 
     #[test]

profiling/src/collections/identifiable/mod.rs

@@ -85,7 +85,7 @@ pub fn into_pprof_iter<T: PprofItem>(
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use super::*;
 
     #[test]

profiling/src/internal/observation/observations.rs

@@ -141,7 +141,7 @@ impl Drop for NonEmptyObservations {
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use super::*;
     use crate::collections::identifiable::*;
     use crate::internal::{LabelSetId, StackTraceId};

profiling/src/internal/observation/trimmed_observation.rs

@@ -123,7 +123,7 @@ impl Drop for TrimmedObservation {
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use super::*;
 
     #[test]

profiling/src/internal/profile.rs

@@ -555,7 +555,7 @@ impl Profile {
 }
 
 #[cfg(test)]
-mod api_test {
+mod api_tests {
     use super::*;
 
     #[test]

profiling/src/pprof/proto.rs

@@ -178,7 +178,7 @@ impl Profile {
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use super::*;
     use prost::Message;
 

sidecar-ffi/tests/sidecar.rs

@@ -28,6 +28,7 @@ fn set_sidecar_per_process() {
 #[test]
 #[cfg(unix)]
 #[cfg_attr(miri, ignore)]
+#[cfg_attr(coverage_nightly, ignore)] // this fails on nightly coverage
 fn test_ddog_ph_file_handling() {
     let fname = CString::new(std::env::temp_dir().join("test_file").to_str().unwrap()).unwrap();
     let mode = CString::new("a+").unwrap();

sidecar/src/log.rs

@@ -389,7 +389,7 @@ pub(crate) fn enable_logging() -> anyhow::Result<()> {
 }
 
 #[cfg(test)]
-mod test {
+mod tests {
     use super::{
         enable_logging, TemporarilyRetainedKeyParser, TemporarilyRetainedMap, MULTI_LOG_FILTER,
     };