Implement apples to apples comparison mode (#2116)

* InvocationCount should be represented with Int64, not Int32

* implement apples-to-apples comparison

Author: adamsitnik

src/BenchmarkDotNet/Configs/ConfigOptions.cs

@@ -40,7 +40,11 @@ public enum ConfigOptions
         /// <summary>
         /// Determines whether to generate msbuild binlogs
         /// </summary>
-        GenerateMSBuildBinLog = 1 << 7
+        GenerateMSBuildBinLog = 1 << 7,
+        /// <summary>
+        /// Performs apples-to-apples comparison for provided benchmarks and jobs. Experimental, will change in the near future!
+        /// </summary>
+        ApplesToApples = 1 << 8
     }
 
     internal static class ConfigOptionsExtensions

src/BenchmarkDotNet/Configs/ManualConfig.cs

@@ -300,6 +300,10 @@ public static ManualConfig Union(IConfig globalConfig, IConfig localConfig)
             return manualConfig;
         }
 
+        internal void RemoveAllJobs() => jobs.Clear();
+
+        internal void RemoveAllDiagnosers() => diagnosers.Clear();
+
         private static TimeSpan GetBuildTimeout(TimeSpan current, TimeSpan other)
             => current == DefaultConfig.Instance.BuildTimeout
                 ? other

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -135,7 +135,7 @@ public bool UseDisassemblyDiagnoser
         public int? MaxIterationCount { get; set; }
 
         [Option("invocationCount", Required = false, HelpText = "Invocation count in a single iteration. By default calculated by the heuristic.")]
-        public int? InvocationCount { get; set; }
+        public long? InvocationCount { get; set; }
 
         [Option("unrollFactor", Required = false, HelpText = "How many times the benchmark method will be invoked per one iteration of a generated loop. 16 by default")]
         public int? UnrollFactor { get; set; }
@@ -152,6 +152,9 @@ public bool UseDisassemblyDiagnoser
         [Option("info", Required = false, Default = false, HelpText = "Print environment information.")]
         public bool PrintInformation { get; set; }
 
+        [Option("apples", Required = false, Default = false, HelpText = "Runs apples-to-apples comparison for specified Jobs.")]
+        public bool ApplesToApples { get; set; }
+
         [Option("list", Required = false, Default = ListBenchmarkCaseMode.Disabled, HelpText = "Prints all of the available benchmark names. Flat/Tree")]
         public ListBenchmarkCaseMode ListBenchmarkCaseMode { get; set; }
 

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -244,6 +244,7 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo
             config.WithOption(ConfigOptions.DisableLogFile, options.DisableLogFile);
             config.WithOption(ConfigOptions.LogBuildOutput, options.LogBuildOutput);
             config.WithOption(ConfigOptions.GenerateMSBuildBinLog, options.GenerateMSBuildBinLog);
+            config.WithOption(ConfigOptions.ApplesToApples, options.ApplesToApples);
 
             if (options.MaxParameterColumnWidth.HasValue)
                 config.WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(options.MaxParameterColumnWidth.Value));

src/BenchmarkDotNet/Jobs/JobExtensions.cs

@@ -170,7 +170,7 @@ public static Job WithHeapAffinitizeMask(this Job job, int heapAffinitizeMask) =
         /// If specified, <see cref="RunMode.IterationTime"/> will be ignored.
         /// If specified, it must be a multiple of <see cref="RunMode.UnrollFactor"/>.
         /// </summary>
-        public static Job WithInvocationCount(this Job job, int count) => job.WithCore(j => j.Run.InvocationCount = count);
+        public static Job WithInvocationCount(this Job job, long count) => job.WithCore(j => j.Run.InvocationCount = count);
 
         /// <summary>
         /// How many times the benchmark method will be invoked per one iteration of a generated loop.

src/BenchmarkDotNet/Jobs/RunMode.cs

@@ -12,7 +12,7 @@ public sealed class RunMode : JobMode<RunMode>
         public static readonly Characteristic<RunStrategy> RunStrategyCharacteristic = Characteristic.Create<RunMode, RunStrategy>(nameof(RunStrategy), RunStrategy.Throughput);
 
         public static readonly Characteristic<int> LaunchCountCharacteristic = CreateCharacteristic<int>(nameof(LaunchCount));
-        public static readonly Characteristic<int> InvocationCountCharacteristic = CreateCharacteristic<int>(nameof(InvocationCount));
+        public static readonly Characteristic<long> InvocationCountCharacteristic = CreateCharacteristic<long>(nameof(InvocationCount));
         public static readonly Characteristic<int> UnrollFactorCharacteristic = CreateCharacteristic<int>(nameof(UnrollFactor));
         public static readonly Characteristic<int> IterationCountCharacteristic = CreateCharacteristic<int>(nameof(IterationCount));
         public static readonly Characteristic<int> MinIterationCountCharacteristic = CreateCharacteristic<int>(nameof(MinIterationCount));
@@ -125,7 +125,7 @@ public TimeInterval IterationTime
         /// If specified, <see cref="IterationTime"/> will be ignored.
         /// If specified, it must be a multiple of <see cref="UnrollFactor"/>.
         /// </summary>
-        public int InvocationCount
+        public long InvocationCount
         {
             get { return InvocationCountCharacteristic[this]; }
             set { InvocationCountCharacteristic[this] = value; }

src/BenchmarkDotNet/Parameters/ParameterInstances.cs

@@ -5,7 +5,7 @@
 
 namespace BenchmarkDotNet.Parameters
 {
-    public class ParameterInstances : IDisposable
+    public class ParameterInstances : IEquatable<ParameterInstances>, IDisposable
     {
         public IReadOnlyList<ParameterInstance> Items { get; }
         public int Count => Items.Count;
@@ -36,5 +36,27 @@ public void Dispose()
         public string PrintInfo => printInfo ?? (printInfo = string.Join("&", Items.Select(p => $"{p.Name}={p.ToDisplayText()}")));
 
         public ParameterInstance GetArgument(string name) => Items.Single(parameter => parameter.IsArgument && parameter.Name == name);
+
+        public bool Equals(ParameterInstances other)
+        {
+            if (other.Count != Count)
+            {
+                return false;
+            }
+
+            for (int i = 0; i < Count; i++)
+            {
+                if (!Items[i].Value.Equals(other[i].Value))
+                {
+                    return false;
+                }
+            }
+
+            return true;
+        }
+
+        public override bool Equals(object obj) => obj is ParameterInstances other && Equals(other);
+
+        public override int GetHashCode() => FolderInfo.GetHashCode();
     }
 }

src/BenchmarkDotNet/Running/BenchmarkSwitcher.cs

@@ -9,9 +9,12 @@
 using BenchmarkDotNet.ConsoleArguments.ListBenchmarks;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Parameters;
 using BenchmarkDotNet.Reports;
 using JetBrains.Annotations;
+using Perfolizer.Mathematics.OutlierDetection;
 
 namespace BenchmarkDotNet.Running
 {
@@ -109,6 +112,11 @@ internal IEnumerable<Summary> RunWithDirtyAssemblyResolveHelper(string[] args, I
                 ? allAvailableTypesWithRunnableBenchmarks
                 : userInteraction.AskUser(allAvailableTypesWithRunnableBenchmarks, logger);
 
+            if (effectiveConfig.Options.HasFlag(ConfigOptions.ApplesToApples))
+            {
+                return ApplesToApples(effectiveConfig, benchmarksToFilter, logger, options);
+            }
+
             var filteredBenchmarks = TypeFilter.Filter(effectiveConfig, benchmarksToFilter);
 
             if (filteredBenchmarks.IsEmpty())
@@ -131,5 +139,74 @@ private static void PrintList(ILogger nonNullLogger, IConfig effectiveConfig, IR
 
             printer.Print(testNames, nonNullLogger);
         }
+
+        private IEnumerable<Summary> ApplesToApples(ManualConfig effectiveConfig, IReadOnlyList<Type> benchmarksToFilter, ILogger logger, CommandLineOptions options)
+        {
+            var jobs = effectiveConfig.GetJobs().ToArray();
+            if (jobs.Length <= 1)
+            {
+                logger.WriteError("To use apples-to-apples comparison you must specify at least two Job objects.");
+                return Array.Empty<Summary>();
+            }
+            var baselineJob = jobs.SingleOrDefault(job => job.Meta.Baseline);
+            if (baselineJob == default)
+            {
+                logger.WriteError("To use apples-to-apples comparison you must specify exactly ONE baseline Job object.");
+                return Array.Empty<Summary>();
+            }
+            else if (jobs.Any(job => !job.Run.HasValue(RunMode.IterationCountCharacteristic)))
+            {
+                logger.WriteError("To use apples-to-apples comparison you must specify the number of iterations in explicit way.");
+                return Array.Empty<Summary>();
+            }
+
+            Job invocationCountJob = baselineJob
+                .WithWarmupCount(1)
+                .WithIterationCount(1)
+                .WithEvaluateOverhead(false);
+
+            ManualConfig invocationCountConfig = ManualConfig.Create(effectiveConfig);
+            invocationCountConfig.RemoveAllJobs();
+            invocationCountConfig.RemoveAllDiagnosers();
+            invocationCountConfig.AddJob(invocationCountJob);
+
+            var invocationCountBenchmarks = TypeFilter.Filter(invocationCountConfig, benchmarksToFilter);
+            if (invocationCountBenchmarks.IsEmpty())
+            {
+                userInteraction.PrintWrongFilterInfo(benchmarksToFilter, logger, options.Filters.ToArray());
+                return Array.Empty<Summary>();
+            }
+
+            logger.WriteLineHeader("Each benchmark is going to be executed just once to get invocation counts.");
+            Summary[] invocationCountSummaries = BenchmarkRunnerClean.Run(invocationCountBenchmarks);
+
+            Dictionary<(Descriptor Descriptor, ParameterInstances Parameters), Measurement> dictionary = invocationCountSummaries
+                .SelectMany(summary => summary.Reports)
+                .ToDictionary(
+                    report => (report.BenchmarkCase.Descriptor, report.BenchmarkCase.Parameters),
+                    report => report.AllMeasurements.Single(measurement => measurement.IsWorkload() && measurement.IterationStage == Engines.IterationStage.Actual));
+
+            int iterationCount = baselineJob.Run.IterationCount;
+            BenchmarkRunInfo[] benchmarksWithoutInvocationCount = TypeFilter.Filter(effectiveConfig, benchmarksToFilter);
+            BenchmarkRunInfo[] benchmarksWithInvocationCount = benchmarksWithoutInvocationCount
+                .Select(benchmarkInfo => new BenchmarkRunInfo(
+                    benchmarkInfo.BenchmarksCases.Select(benchmark =>
+                        new BenchmarkCase(
+                            benchmark.Descriptor,
+                            benchmark.Job
+                                .WithIterationCount(iterationCount)
+                                .WithEvaluateOverhead(false)
+                                .WithWarmupCount(1)
+                                .WithOutlierMode(OutlierMode.DontRemove)
+                                .WithInvocationCount(dictionary[(benchmark.Descriptor, benchmark.Parameters)].Operations)
+                                .WithUnrollFactor(dictionary[(benchmark.Descriptor, benchmark.Parameters)].Operations % 16 == 0 ? 16 : 1),
+                            benchmark.Parameters,
+                            benchmark.Config)).ToArray(),
+                    benchmarkInfo.Type, benchmarkInfo.Config))
+                .ToArray();
+
+            logger.WriteLineHeader("Actual benchmarking is going to happen now!");
+            return BenchmarkRunnerClean.Run(benchmarksWithInvocationCount);
+        }
     }
 }

src/BenchmarkDotNet/Running/Descriptor.cs

@@ -8,7 +8,7 @@
 
 namespace BenchmarkDotNet.Running
 {
-    public class Descriptor
+    public class Descriptor : IEquatable<Descriptor>
     {
         public Type Type { get; }
         public MethodInfo WorkloadMethod { get; }
@@ -70,5 +70,11 @@ private static string FormatDescription([CanBeNull] string description)
         public bool HasCategory(string category) => Categories.Any(c => c.EqualsWithIgnoreCase(category));
 
         public string GetFilterName() => $"{Type.GetCorrectCSharpTypeName(includeGenericArgumentsNamespace: false)}.{WorkloadMethod.Name}";
+
+        public bool Equals(Descriptor other) => GetFilterName().Equals(other.GetFilterName());
+
+        public override bool Equals(object obj) => obj is Descriptor && Equals((Descriptor)obj);
+
+        public override int GetHashCode() => GetFilterName().GetHashCode();
     }
 }

src/BenchmarkDotNet/Validators/RunModeValidator.cs

@@ -28,7 +28,7 @@ public IEnumerable<ValidationError> Validate(ValidationParameters validationPara
                 }
                 else if (run.HasValue(RunMode.InvocationCountCharacteristic))
                 {
-                    int invocationCount = run.InvocationCount;
+                    long invocationCount = run.InvocationCount;
                     if (invocationCount % unrollFactor != 0)
                     {
                         string message = $"Specified InvocationCount ({invocationCount}) must be a multiple of UnrollFactor ({unrollFactor})";