diff --git a/docs/articles/configs/configoptions.md b/docs/articles/configs/configoptions.md index fe46284bf2..66705ddd65 100644 --- a/docs/articles/configs/configoptions.md +++ b/docs/articles/configs/configoptions.md @@ -10,6 +10,7 @@ The config options let you customize some behavior of BenchmarkDotNet - mainly r Available config options are: * `ConfigOptions.Default` - No configuration option is set - this is the default. +* `ConfigOptions.GenerateAndBuildOnly` - Only run toolchain generator and builder without executor. Then performs normal cleanup. * `ConfigOptions.KeepBenchmarkFiles` - All auto-generated files should be kept after running the benchmarks (by default they are removed). * `ConfigOptions.JoinSummary` - All benchmarks results should be joined into a single summary (by default we have a summary per type). * `ConfigOptions.StopOnFirstError` - Benchmarking should be stopped after the first error (by default it's not). diff --git a/src/BenchmarkDotNet/Attributes/GenerateAndBuildOnlyAttribute.cs b/src/BenchmarkDotNet/Attributes/GenerateAndBuildOnlyAttribute.cs new file mode 100644 index 0000000000..dda5b0d427 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/GenerateAndBuildOnlyAttribute.cs @@ -0,0 +1,21 @@ +using System; +using BenchmarkDotNet.Configs; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Attributes +{ + /// + /// determines if benchmark should be run after being generated and built + /// + [PublicAPI] + [AttributeUsage(AttributeTargets.Class)] + public class GenerateAndBuildOnlyAttribute : Attribute, IConfigSource + { + public IConfig Config { get; } + + public GenerateAndBuildOnlyAttribute(bool value = true) + { + Config = ManualConfig.CreateEmpty().WithOption(ConfigOptions.GenerateAndBuildOnly, value); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs index fa0920c7f9..050438a1f2 100644 --- a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs @@ -80,6 +80,11 @@ public static class ConfigExtensions /// [PublicAPI] public static IConfig KeepBenchmarkFiles(this IConfig config, bool value = true) => config.WithOption(ConfigOptions.KeepBenchmarkFiles, value); + /// + /// determines if we skip executing after running generator and builder + /// + [PublicAPI] public static IConfig GenerateAndBuildOnly(this IConfig config, bool value = true) => config.WithOption(ConfigOptions.GenerateAndBuildOnly, value); + /// /// determines if the exported result files should not be overwritten (be default they are overwritten) /// diff --git a/src/BenchmarkDotNet/Configs/ConfigOptions.cs b/src/BenchmarkDotNet/Configs/ConfigOptions.cs index c1ad75d642..e7c3e6b2fb 100644 --- a/src/BenchmarkDotNet/Configs/ConfigOptions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigOptions.cs @@ -48,7 +48,11 @@ public enum ConfigOptions /// /// Continue the execution if the last run was stopped. /// - Resume = 1 << 9 + Resume = 1 << 9, + /// + /// Don't run the benchmark after it's generated and built. + /// + GenerateAndBuildOnly = 1 << 10 } internal static class ConfigOptionsExtensions diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index e3aed1fedd..fc4230e07d 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -86,6 +86,9 @@ public bool UseDisassemblyDiagnoser [Option("keepFiles", Required = false, Default = false, HelpText = "Determines if all auto-generated files should be kept or removed after running the benchmarks.")] public bool KeepBenchmarkFiles { get; set; } + [Option("generateAndBuildOnly", Required = false, Default = false, HelpText = "Determines if we only run generator and builder without executor.")] + public bool GenerateAndBuildOnly { get; set; } + [Option("noOverwrite", Required = false, Default = false, HelpText = "Determines if the exported result files should not be overwritten (be default they are overwritten).")] public bool DontOverwriteResults { get; set; } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index e4b730aca3..35ed39172b 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -242,6 +242,7 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo config.WithOption(ConfigOptions.JoinSummary, options.Join); config.WithOption(ConfigOptions.KeepBenchmarkFiles, options.KeepBenchmarkFiles); + config.WithOption(ConfigOptions.GenerateAndBuildOnly, options.GenerateAndBuildOnly); config.WithOption(ConfigOptions.DontOverwriteResults, options.DontOverwriteResults); config.WithOption(ConfigOptions.StopOnFirstError, options.StopOnFirstError); config.WithOption(ConfigOptions.DisableLogFile, options.DisableLogFile); diff --git a/src/BenchmarkDotNet/Reports/BenchmarkReport.cs b/src/BenchmarkDotNet/Reports/BenchmarkReport.cs index b63c5221be..29a80ac244 100644 --- a/src/BenchmarkDotNet/Reports/BenchmarkReport.cs +++ b/src/BenchmarkDotNet/Reports/BenchmarkReport.cs @@ -47,6 +47,16 @@ public BenchmarkReport( ?? (IReadOnlyDictionary)ImmutableDictionary.Empty; } + internal BenchmarkReport(BenchmarkCase benchmarkCase, GenerateResult generateResult, BuildResult buildResult) + { + BenchmarkCase = benchmarkCase; + GenerateResult = generateResult; + BuildResult = buildResult; + ExecuteResults = Array.Empty(); + AllMeasurements = ImmutableList.Empty; + Metrics = ImmutableDictionary.Empty; + } + public override string ToString() => $"{BenchmarkCase.DisplayInfo}, {AllMeasurements.Count} runs"; public IReadOnlyList GetResultRuns() => AllMeasurements.Where(r => r.Is(IterationMode.Workload, IterationStage.Result)).ToList(); diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 5bf2e5ed03..06f30e09e0 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -196,7 +196,15 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, if (!config.Options.IsSet(ConfigOptions.KeepBenchmarkFiles)) artifactsToCleanup.AddRange(buildResult.ArtifactsToCleanup); - var report = RunCore(benchmark, info.benchmarkId, logger, resolver, buildResult); + BenchmarkReport report; + if (config.Options.IsSet(ConfigOptions.GenerateAndBuildOnly)) + { + report = new BenchmarkReport(benchmark, buildResult, buildResult); + reports.Add(report); + continue; + } + + report = RunCore(benchmark, info.benchmarkId, logger, resolver, buildResult); if (report.AllMeasurements.Any(m => m.Operations == 0)) throw new InvalidOperationException("An iteration with 'Operations == 0' detected"); reports.Add(report);