Make benchmark printer and runner correctly deal with 0 time

CHANGELOG.md

@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Bugfixes (User Facing)
+* fix a bug where if times were supplied as `0` instead of `0.0` we'd sometimes gather a single measurement
+
 ## 1.4.0 (2025-04-14)
 
 Some nice features (`pre_check: :all_same` is cool) along with adding support for some new stuff (`tprof`) and fixing some bugs.

lib/benchee/benchmark/runner.ex

@@ -17,6 +17,7 @@ defmodule Benchee.Benchmark.Runner do
   }
 
   @no_input Benchmark.no_input()
+  @zero_values [0, 0.0]
 
   @doc """
   Executes the benchmarks defined before by first running the defined functions
@@ -201,7 +202,8 @@ defmodule Benchee.Benchmark.Runner do
     Enum.map(reductions, &(&1 - offset))
   end
 
-  defp run_reductions_benchmark(_, %ScenarioContext{config: %{reduction_time: +0.0}}) do
+  defp run_reductions_benchmark(_, %ScenarioContext{config: %{reduction_time: time}})
+       when time in @zero_values do
     []
   end
 
@@ -224,7 +226,8 @@ defmodule Benchee.Benchmark.Runner do
     do_benchmark(scenario, new_context, Collect.Reductions, [])
   end
 
-  defp run_memory_benchmark(_, %ScenarioContext{config: %{memory_time: +0.0}}) do
+  defp run_memory_benchmark(_, %ScenarioContext{config: %{memory_time: time}})
+       when time in @zero_values do
     []
   end
 
@@ -249,7 +252,7 @@ defmodule Benchee.Benchmark.Runner do
 
   @spec measure_runtimes(Scenario.t(), ScenarioContext.t(), number, boolean) :: [number]
   defp measure_runtimes(scenario, context, run_time, fast_warning)
-  defp measure_runtimes(_, _, +0.0, _), do: []
+  defp measure_runtimes(_, _, time, _) when time in @zero_values, do: []
 
   defp measure_runtimes(scenario, scenario_context, run_time, fast_warning) do
     end_time = current_time() + run_time

lib/benchee/output/benchmark_printer.ex

@@ -1,7 +1,9 @@
 defmodule Benchee.Output.BenchmarkPrinter do
   @moduledoc false
 
-  alias Benchee.{Benchmark, Conversion.Duration, System}
+  alias Benchee.Benchmark
+  alias Benchee.Conversion.Duration
+  alias Benchee.System
 
   @doc """
   Shown when you try benchmark an evaluated function.
@@ -96,11 +98,14 @@ defmodule Benchee.Output.BenchmarkPrinter do
   """
   def benchmarking(_, _, %{print: %{benchmarking: false}}), do: nil
 
-  def benchmarking(_, _, %{time: +0.0, warmup: +0.0, memory_time: +0.0, reduction_time: +0.0}),
-    do: nil
+  def benchmarking(name, input_name, config) do
+    time_configs = [config.time, config.warmup, config.memory_time, config.reduction_time]
 
-  def benchmarking(name, input_name, _config) do
-    IO.puts("Benchmarking #{name}#{input_information(input_name)} ...")
+    if Enum.all?(time_configs, fn time -> time == 0 end) do
+      nil
+    else
+      IO.puts("Benchmarking #{name}#{input_information(input_name)} ...")
+    end
   end
 
   @no_input Benchmark.no_input()

test/benchee/benchmark/runner_test.exs

@@ -264,10 +264,21 @@ defmodule Benchee.Benchmark.RunnerTest do
       end)
     end
 
+    test "never calls the function if warmup and time are 0.0" do
+      ref = self()
+
+      %Suite{configuration: %{time: 0.0, warmup: 0.0, memory_time: 0.0, reduction_time: 0.0}}
+      |> test_suite
+      |> Benchmark.benchmark("", fn -> send(ref, :called) end)
+      |> Benchmark.collect(FakeBenchmarkPrinter)
+
+      refute_receive :called
+    end
+
     test "never calls the function if warmup and time are 0" do
       ref = self()
 
-      %Suite{configuration: %{time: 0.0, warmup: 0.0}}
+      %Suite{configuration: %{time: 0, warmup: 0, memory_time: 0, reduction_time: 0}}
       |> test_suite
       |> Benchmark.benchmark("", fn -> send(ref, :called) end)
       |> Benchmark.collect(FakeBenchmarkPrinter)

test/benchee/output/benchmark_printer_test.exs

@@ -1,7 +1,11 @@
 defmodule Benchee.Output.BenchmarkPrintertest do
   use ExUnit.Case, async: true
 
-  alias Benchee.{Benchmark, Configuration, Scenario, System}
+  alias Benchee.Benchmark
+  alias Benchee.Benchmark.BenchmarkConfig
+  alias Benchee.Configuration
+  alias Benchee.Scenario
+  alias Benchee.System
 
   import ExUnit.CaptureIO
   import Benchee.Output.BenchmarkPrinter
@@ -125,7 +129,7 @@ defmodule Benchee.Output.BenchmarkPrintertest do
     test "prints information that it's currently benchmarking without input" do
       output =
         capture_io(fn ->
-          benchmarking("Something", @no_input, %{})
+          benchmarking("Something", @no_input, %BenchmarkConfig{})
         end)
 
       assert output =~ ~r/Benchmarking.+Something/i
@@ -134,7 +138,7 @@ defmodule Benchee.Output.BenchmarkPrintertest do
     test "prints information that it's currently benchmarking with input" do
       output =
         capture_io(fn ->
-          benchmarking("Something", "great input", %{})
+          benchmarking("Something", "great input", %BenchmarkConfig{})
         end)
 
       assert output =~ ~r/Benchmarking.+Something with input great input/i
@@ -143,19 +147,34 @@ defmodule Benchee.Output.BenchmarkPrintertest do
     test "doesn't print if it's deactivated" do
       output =
         capture_io(fn ->
-          benchmarking("A", "some", %{print: %{benchmarking: false}})
+          benchmarking("A", "some", %BenchmarkConfig{print: %{benchmarking: false}})
         end)
 
       assert output == ""
     end
 
-    test "doesn't print if all times are set to 0" do
+    test "doesn't print if all times are set to 0.0" do
       output =
         capture_io(fn ->
-          benchmarking("Never", "don't care", %Configuration{
+          benchmarking("Never", "don't care", %BenchmarkConfig{
             time: 0.0,
             warmup: 0.0,
-            memory_time: 0.0
+            memory_time: 0.0,
+            reduction_time: 0.0
+          })
+        end)
+
+      assert output == ""
+    end
+
+    test "doesn't print if all times are set to 0" do
+      output =
+        capture_io(fn ->
+          benchmarking("Never", "don't care", %BenchmarkConfig{
+            time: 0,
+            warmup: 0,
+            memory_time: 0,
+            reduction_time: 0
           })
         end)