Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

More extensive tests #491

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

More extensive tests #491

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b777ece
Make extensive a field rather than enum variant
tgross35 Feb 5, 2025
0ee064d
Add more extensive tests
tgross35 Feb 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/libm-test/benches/icount.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ macro_rules! icount_benches {
let mut ctx = CheckCtx::new(
Op::IDENTIFIER,
CheckBasis::None,
GeneratorKind::QuickSpaced
GeneratorKind::Spaced
);
ctx.override_iterations(BENCH_ITER_ITEMS);
let ret = spaced::get_test_cases::<Op>(&ctx).0.collect::<Vec<_>>();
Expand Down
2 changes: 1 addition & 1 deletion crates/libm-test/benches/random.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ where

let ctx = CheckCtx::new(Op::IDENTIFIER, CheckBasis::Musl, GeneratorKind::Random);
let benchvec: Vec<_> =
random::get_test_cases::<Op::RustArgs>(&ctx).0.take(BENCH_ITER_ITEMS).collect();
random::get_test_cases::<Op>(ctx.clone()).0.take(BENCH_ITER_ITEMS).collect();

// Perform a sanity check that we are benchmarking the same thing
// Don't test against musl if it is not available
Expand Down
12 changes: 9 additions & 3 deletions crates/libm-test/examples/plot_domains.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,21 @@ where
Op: MathOp<FTy = f32, RustArgs = (f32,)>,
Op::RustArgs: SpacedInput<Op>,
{
let mut ctx = CheckCtx::new(Op::IDENTIFIER, CheckBasis::Mpfr, GeneratorKind::QuickSpaced);
plot_one_generator(out_dir, &ctx, "logspace", config, spaced::get_test_cases::<Op>(&ctx).0);
let mut ctx = CheckCtx::new(Op::IDENTIFIER, CheckBasis::Mpfr, GeneratorKind::Spaced);
plot_one_generator(
out_dir,
&ctx,
"logspace",
config,
spaced::get_test_cases::<Op>(ctx.clone()).0,
);
ctx.gen_kind = GeneratorKind::EdgeCases;
plot_one_generator(
out_dir,
&ctx,
"edge_cases",
config,
edge_cases::get_test_cases::<Op>(&ctx).0,
edge_cases::get_test_cases::<Op>(ctx.clone()).0,
);
}

Expand Down
24 changes: 14 additions & 10 deletions crates/libm-test/src/gen/edge_cases.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ use libm::support::{Float, Int};
use crate::domain::get_domain;
use crate::gen::KnownSize;
use crate::run_cfg::{check_near_count, check_point_count};
use crate::{CheckCtx, FloatExt, MathOp, test_log};
use crate::{CheckCtx, FloatExt, GeneratorKind, MathOp, test_log};

/// Generate a sequence of edge cases, e.g. numbers near zeroes and infiniteis.
pub trait EdgeCaseInput<Op> {
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self> + Send, u64);
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self> + Send, u64);
}

/// Create a list of values around interesting points (infinities, zeroes, NaNs).
Expand Down Expand Up @@ -140,7 +140,8 @@ macro_rules! impl_edge_case_input {
where
Op: MathOp<RustArgs = Self, FTy = $fty>,
{
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let ctx = &ctx;
let (iter0, steps0) = float_edge_cases::<Op>(ctx, 0);
let iter0 = iter0.map(|v| (v,));
(iter0, steps0)
Expand All @@ -151,7 +152,8 @@ macro_rules! impl_edge_case_input {
where
Op: MathOp<RustArgs = Self, FTy = $fty>,
{
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let ctx = &ctx;
let (iter0, steps0) = float_edge_cases::<Op>(ctx, 0);
let (iter1, steps1) = float_edge_cases::<Op>(ctx, 1);
let iter =
Expand All @@ -165,7 +167,8 @@ macro_rules! impl_edge_case_input {
where
Op: MathOp<RustArgs = Self, FTy = $fty>,
{
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let ctx = &ctx;
let (iter0, steps0) = float_edge_cases::<Op>(ctx, 0);
let (iter1, steps1) = float_edge_cases::<Op>(ctx, 1);
let (iter2, steps2) = float_edge_cases::<Op>(ctx, 2);
Expand All @@ -185,7 +188,8 @@ macro_rules! impl_edge_case_input {
where
Op: MathOp<RustArgs = Self, FTy = $fty>,
{
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let ctx = &ctx;
let (iter0, steps0) = int_edge_cases(ctx, 0);
let (iter1, steps1) = float_edge_cases::<Op>(ctx, 1);

Expand All @@ -201,7 +205,8 @@ macro_rules! impl_edge_case_input {
where
Op: MathOp<RustArgs = Self, FTy = $fty>,
{
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let ctx = &ctx;
let (iter0, steps0) = float_edge_cases::<Op>(ctx, 0);
let (iter1, steps1) = int_edge_cases(ctx, 1);

Expand All @@ -222,13 +227,12 @@ impl_edge_case_input!(f64);
#[cfg(f128_enabled)]
impl_edge_case_input!(f128);

pub fn get_test_cases<Op>(
ctx: &CheckCtx,
) -> (impl Iterator<Item = Op::RustArgs> + Send + use<'_, Op>, u64)
pub fn get_test_cases<Op>(ctx: CheckCtx) -> (impl Iterator<Item = Op::RustArgs> + Send, u64)
where
Op: MathOp,
Op::RustArgs: EdgeCaseInput<Op>,
{
assert_eq!(ctx.gen_kind, GeneratorKind::EdgeCases);
let (iter, count) = Op::RustArgs::get_cases(ctx);

// Wrap in `KnownSize` so we get an assertion if the cuunt is wrong.
Expand Down
30 changes: 19 additions & 11 deletions crates/libm-test/src/gen/random.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha8Rng;

use super::KnownSize;
use crate::CheckCtx;
use crate::run_cfg::{int_range, iteration_count};
use crate::{CheckCtx, GeneratorKind, MathOp};

pub(crate) const SEED_ENV: &str = "LIBM_SEED";

Expand All @@ -27,7 +27,7 @@ pub(crate) static SEED: LazyLock<[u8; 32]> = LazyLock::new(|| {

/// Generate a sequence of random values of this type.
pub trait RandomInput: Sized {
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self> + Send, u64);
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self> + Send, u64);
}

/// Generate a sequence of deterministically random floats.
Expand All @@ -51,15 +51,17 @@ fn random_ints(count: u64, range: RangeInclusive<i32>) -> impl Iterator<Item = i
macro_rules! impl_random_input {
($fty:ty) => {
impl RandomInput for ($fty,) {
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let ctx = &ctx;
let count = iteration_count(ctx, 0);
let iter = random_floats(count).map(|f: $fty| (f,));
(iter, count)
}
}

impl RandomInput for ($fty, $fty) {
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let ctx = &ctx;
let count0 = iteration_count(ctx, 0);
let count1 = iteration_count(ctx, 1);
let iter = random_floats(count0)
Expand All @@ -69,7 +71,8 @@ macro_rules! impl_random_input {
}

impl RandomInput for ($fty, $fty, $fty) {
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let ctx = &ctx;
let count0 = iteration_count(ctx, 0);
let count1 = iteration_count(ctx, 1);
let count2 = iteration_count(ctx, 2);
Expand All @@ -83,7 +86,8 @@ macro_rules! impl_random_input {
}

impl RandomInput for (i32, $fty) {
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let ctx = &ctx;
let count0 = iteration_count(ctx, 0);
let count1 = iteration_count(ctx, 1);
let range0 = int_range(ctx, 0);
Expand All @@ -94,7 +98,8 @@ macro_rules! impl_random_input {
}

impl RandomInput for ($fty, i32) {
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let ctx = &ctx;
let count0 = iteration_count(ctx, 0);
let count1 = iteration_count(ctx, 1);
let range1 = int_range(ctx, 1);
Expand All @@ -115,10 +120,13 @@ impl_random_input!(f64);
impl_random_input!(f128);

/// Create a test case iterator.
pub fn get_test_cases<RustArgs: RandomInput>(
ctx: &CheckCtx,
) -> (impl Iterator<Item = RustArgs> + Send + use<'_, RustArgs>, u64) {
let (iter, count) = RustArgs::get_cases(ctx);
pub fn get_test_cases<Op>(ctx: CheckCtx) -> (impl Iterator<Item = Op::RustArgs> + Send, u64)
where
Op: MathOp,
Op::RustArgs: RandomInput,
{
assert_eq!(ctx.gen_kind, GeneratorKind::Random);
let (iter, count) = Op::RustArgs::get_cases(ctx);

// Wrap in `KnownSize` so we get an assertion if the cuunt is wrong.
(KnownSize::new(iter, count), count)
Expand Down
59 changes: 29 additions & 30 deletions crates/libm-test/src/gen/spaced.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ use libm::support::{Float, MinInt};
use crate::domain::get_domain;
use crate::op::OpITy;
use crate::run_cfg::{int_range, iteration_count};
use crate::{CheckCtx, MathOp, linear_ints, logspace};
use crate::{CheckCtx, GeneratorKind, MathOp, linear_ints, logspace};

/// Generate a sequence of inputs that eiher cover the domain in completeness (for smaller float
/// types and single argument functions) or provide evenly spaced inputs across the domain with
/// approximately `u32::MAX` total iterations.
pub trait SpacedInput<Op> {
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self> + Send, u64);
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self> + Send, u64);
}

/// Construct an iterator from `logspace` and also calculate the total number of steps expected
Expand Down Expand Up @@ -87,8 +87,8 @@ macro_rules! impl_spaced_input {
where
Op: MathOp<RustArgs = Self, FTy = $fty>,
{
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let max_steps0 = iteration_count(ctx, 0);
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let max_steps0 = iteration_count(&ctx, 0);
// `f16` and `f32` can have exhaustive tests.
match value_count::<Op::FTy>() {
Some(steps0) if steps0 <= max_steps0 => {
Expand All @@ -97,7 +97,7 @@ macro_rules! impl_spaced_input {
(EitherIter::A(iter0), steps0)
}
_ => {
let (iter0, steps0) = logspace_steps::<Op>(ctx, 0, max_steps0);
let (iter0, steps0) = logspace_steps::<Op>(&ctx, 0, max_steps0);
let iter0 = iter0.map(|v| (v,));
(EitherIter::B(iter0), steps0)
}
Expand All @@ -109,9 +109,9 @@ macro_rules! impl_spaced_input {
where
Op: MathOp<RustArgs = Self, FTy = $fty>,
{
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let max_steps0 = iteration_count(ctx, 0);
let max_steps1 = iteration_count(ctx, 1);
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let max_steps0 = iteration_count(&ctx, 0);
let max_steps1 = iteration_count(&ctx, 1);
// `f16` can have exhaustive tests.
match value_count::<Op::FTy>() {
Some(count) if count <= max_steps0 && count <= max_steps1 => {
Expand All @@ -120,8 +120,8 @@ macro_rules! impl_spaced_input {
(EitherIter::A(iter), count.checked_mul(count).unwrap())
}
_ => {
let (iter0, steps0) = logspace_steps::<Op>(ctx, 0, max_steps0);
let (iter1, steps1) = logspace_steps::<Op>(ctx, 1, max_steps1);
let (iter0, steps0) = logspace_steps::<Op>(&ctx, 0, max_steps0);
let (iter1, steps1) = logspace_steps::<Op>(&ctx, 1, max_steps1);
let iter = iter0.flat_map(move |first| {
iter1.clone().map(move |second| (first, second))
});
Expand All @@ -136,10 +136,10 @@ macro_rules! impl_spaced_input {
where
Op: MathOp<RustArgs = Self, FTy = $fty>,
{
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let max_steps0 = iteration_count(ctx, 0);
let max_steps1 = iteration_count(ctx, 1);
let max_steps2 = iteration_count(ctx, 2);
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let max_steps0 = iteration_count(&ctx, 0);
let max_steps1 = iteration_count(&ctx, 1);
let max_steps2 = iteration_count(&ctx, 2);
// `f16` can be exhaustive tested if `LIBM_EXTENSIVE_TESTS` is incresed.
match value_count::<Op::FTy>() {
Some(count)
Expand All @@ -153,9 +153,9 @@ macro_rules! impl_spaced_input {
(EitherIter::A(iter), count.checked_pow(3).unwrap())
}
_ => {
let (iter0, steps0) = logspace_steps::<Op>(ctx, 0, max_steps0);
let (iter1, steps1) = logspace_steps::<Op>(ctx, 1, max_steps1);
let (iter2, steps2) = logspace_steps::<Op>(ctx, 2, max_steps2);
let (iter0, steps0) = logspace_steps::<Op>(&ctx, 0, max_steps0);
let (iter1, steps1) = logspace_steps::<Op>(&ctx, 1, max_steps1);
let (iter2, steps2) = logspace_steps::<Op>(&ctx, 2, max_steps2);

let iter = iter0
.flat_map(move |first| iter1.clone().map(move |second| (first, second)))
Expand All @@ -175,10 +175,10 @@ macro_rules! impl_spaced_input {
where
Op: MathOp<RustArgs = Self, FTy = $fty>,
{
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let range0 = int_range(ctx, 0);
let max_steps0 = iteration_count(ctx, 0);
let max_steps1 = iteration_count(ctx, 1);
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let range0 = int_range(&ctx, 0);
let max_steps0 = iteration_count(&ctx, 0);
let max_steps1 = iteration_count(&ctx, 1);
match value_count::<Op::FTy>() {
Some(count1) if count1 <= max_steps1 => {
let (iter0, steps0) = linear_ints(range0, max_steps0);
Expand All @@ -188,7 +188,7 @@ macro_rules! impl_spaced_input {
}
_ => {
let (iter0, steps0) = linear_ints(range0, max_steps0);
let (iter1, steps1) = logspace_steps::<Op>(ctx, 1, max_steps1);
let (iter1, steps1) = logspace_steps::<Op>(&ctx, 1, max_steps1);

let iter = iter0.flat_map(move |first| {
iter1.clone().map(move |second| (first, second))
Expand All @@ -205,10 +205,10 @@ macro_rules! impl_spaced_input {
where
Op: MathOp<RustArgs = Self, FTy = $fty>,
{
fn get_cases(ctx: &CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let max_steps0 = iteration_count(ctx, 0);
let range1 = int_range(ctx, 1);
let max_steps1 = iteration_count(ctx, 1);
fn get_cases(ctx: CheckCtx) -> (impl Iterator<Item = Self>, u64) {
let max_steps0 = iteration_count(&ctx, 0);
let range1 = int_range(&ctx, 1);
let max_steps1 = iteration_count(&ctx, 1);
match value_count::<Op::FTy>() {
Some(count0) if count0 <= max_steps0 => {
let (iter1, steps1) = linear_ints(range1, max_steps1);
Expand All @@ -218,7 +218,7 @@ macro_rules! impl_spaced_input {
(EitherIter::A(iter), count0.checked_mul(steps1).unwrap())
}
_ => {
let (iter0, steps0) = logspace_steps::<Op>(ctx, 0, max_steps0);
let (iter0, steps0) = logspace_steps::<Op>(&ctx, 0, max_steps0);
let (iter1, steps1) = linear_ints(range1, max_steps1);

let iter = iter0.flat_map(move |first| {
Expand All @@ -242,12 +242,11 @@ impl_spaced_input!(f64);
impl_spaced_input!(f128);

/// Create a test case iterator for extensive inputs. Also returns the total test case count.
pub fn get_test_cases<Op>(
ctx: &CheckCtx,
) -> (impl Iterator<Item = Op::RustArgs> + Send + use<'_, Op>, u64)
pub fn get_test_cases<Op>(ctx: CheckCtx) -> (impl Iterator<Item = Op::RustArgs> + Send, u64)
where
Op: MathOp,
Op::RustArgs: SpacedInput<Op>,
{
assert_eq!(ctx.gen_kind, GeneratorKind::Spaced);
Op::RustArgs::get_cases(ctx)
}
Loading
Loading