From fcf693df85aae9d0ff912ec692ff3fee4bd22b3d Mon Sep 17 00:00:00 2001 From: Francisco Loureiro Date: Mon, 17 Oct 2022 21:55:22 +0000 Subject: [PATCH 1/4] add quiet mode setting --- .../Attributes/QuietModeAttribute.cs | 21 +++++++++++++++++++ .../Configs/ConfigExtensions.cs | 5 +++++ src/BenchmarkDotNet/Configs/ConfigOptions.cs | 6 +++++- .../ConsoleArguments/CommandLineOptions.cs | 3 +++ .../ConsoleArguments/ConfigParser.cs | 1 + 5 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 src/BenchmarkDotNet/Attributes/QuietModeAttribute.cs diff --git a/src/BenchmarkDotNet/Attributes/QuietModeAttribute.cs b/src/BenchmarkDotNet/Attributes/QuietModeAttribute.cs new file mode 100644 index 0000000000..c40e890c20 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/QuietModeAttribute.cs @@ -0,0 +1,21 @@ +using BenchmarkDotNet.Configs; +using JetBrains.Annotations; +using System; + +namespace BenchmarkDotNet.Attributes +{ + /// + /// determines if all auto-generated files should be kept or removed after running the benchmarks + /// + [PublicAPI] + [AttributeUsage(AttributeTargets.Class)] + public class QuietModeAttribute : Attribute, IConfigSource + { + public IConfig Config { get; } + + public QuietModeAttribute(bool value = false) + { + Config = ManualConfig.CreateEmpty().WithOption(ConfigOptions.QuietMode, value); + } + } +} diff --git a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs index fa0920c7f9..41c40be08a 100644 --- a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs @@ -75,6 +75,11 @@ public static class ConfigExtensions [PublicAPI] public static ManualConfig WithUnionRule(this IConfig config, ConfigUnionRule unionRule) => config.With(m => m.WithUnionRule(unionRule)); [PublicAPI] public static ManualConfig WithCultureInfo(this IConfig config, CultureInfo cultureInfo) => config.With(m => m.CultureInfo = cultureInfo); + /// + /// Run benchmars in quiet mode. + /// + [PublicAPI] public static IConfig QuietMode(this IConfig config, bool value = true) => config.WithOption(ConfigOptions.QuietMode, value); + /// /// determines if all auto-generated files should be kept or removed after running the benchmarks /// diff --git a/src/BenchmarkDotNet/Configs/ConfigOptions.cs b/src/BenchmarkDotNet/Configs/ConfigOptions.cs index 29e9e1af6f..201ee74e58 100644 --- a/src/BenchmarkDotNet/Configs/ConfigOptions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigOptions.cs @@ -44,7 +44,11 @@ public enum ConfigOptions /// /// Performs apples-to-apples comparison for provided benchmarks and jobs. Experimental, will change in the near future! /// - ApplesToApples = 1 << 8 + ApplesToApples = 1 << 8, + /// + /// Run benchmars in quiet mode. + /// + QuietMode = 1 << 9 } internal static class ConfigOptionsExtensions diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index 8a52323ad2..3bd76219da 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -155,6 +155,9 @@ public bool UseDisassemblyDiagnoser [Option("apples", Required = false, Default = false, HelpText = "Runs apples-to-apples comparison for specified Jobs.")] public bool ApplesToApples { get; set; } + [Option("quiet", Required = false, Default = false, HelpText = "Run benchmars in quiet mode.")] + public bool QuietMode { get; set; } + [Option("list", Required = false, Default = ListBenchmarkCaseMode.Disabled, HelpText = "Prints all of the available benchmark names. Flat/Tree")] public ListBenchmarkCaseMode ListBenchmarkCaseMode { get; set; } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 5f5203cc30..1085d2f1af 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -245,6 +245,7 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo config.WithOption(ConfigOptions.LogBuildOutput, options.LogBuildOutput); config.WithOption(ConfigOptions.GenerateMSBuildBinLog, options.GenerateMSBuildBinLog); config.WithOption(ConfigOptions.ApplesToApples, options.ApplesToApples); + config.WithOption(ConfigOptions.QuietMode, options.QuietMode); if (options.MaxParameterColumnWidth.HasValue) config.WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(options.MaxParameterColumnWidth.Value)); From 80d2f38dc4d6f2676c66a916229ccc97ac054b62 Mon Sep 17 00:00:00 2001 From: Francisco Loureiro Date: Mon, 17 Oct 2022 22:54:57 +0000 Subject: [PATCH 2/4] use quiet logger --- .../Running/BenchmarkRunnerClean.cs | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 7e5b56c118..dd299c1f75 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -150,7 +150,9 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, logger.WriteLineInfo($"// {benchmark.DisplayInfo}"); logger.WriteLine(); - using (var powerManagementApplier = new PowerManagementApplier(logger)) + ILogger quietLogger = !config.Options.IsSet(ConfigOptions.QuietMode) ? logger : NullLogger.Instance; + + using (var powerManagementApplier = new PowerManagementApplier(quietLogger)) { bool stop = false; @@ -213,8 +215,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, logger.WriteLine(); benchmarksToRunCount -= stop ? benchmarks.Length - i : 1; - - LogProgress(logger, in runsChronometer, totalBenchmarkCount, benchmarksToRunCount); + LogProgress(benchmarksToRunCount == 0 ? logger : quietLogger, in runsChronometer, totalBenchmarkCount, benchmarksToRunCount); } } @@ -227,7 +228,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, logFilePath, runEnd.GetTimeSpan() - runStart.GetTimeSpan(), cultureInfo, - Validate(new[] {benchmarkRunInfo }, NullLogger.Instance), // validate them once again, but don't print the output + Validate(new[] { benchmarkRunInfo }, NullLogger.Instance), // validate them once again, but don't print the output config.GetColumnHidingRules().ToImmutableArray()); } @@ -381,9 +382,10 @@ private static BuildResult Build(BuildPartition buildPartition, string rootArtif private static BenchmarkReport RunCore(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, IResolver resolver, BuildResult buildResult) { var toolchain = benchmarkCase.GetToolchain(); + ILogger quietLogger = !benchmarkCase.Config.Options.IsSet(ConfigOptions.QuietMode) ? logger : NullLogger.Instance; - logger.WriteLineHeader("// **************************"); - logger.WriteLineHeader("// Benchmark: " + benchmarkCase.DisplayInfo); + quietLogger.WriteLineHeader("// **************************"); + quietLogger.WriteLineHeader("// Benchmark: " + benchmarkCase.DisplayInfo); var (success, executeResults, metrics) = Execute(logger, benchmarkCase, benchmarkId, toolchain, buildResult, resolver); @@ -395,8 +397,9 @@ private static (bool success, List executeResults, List m { var executeResults = new List(); var metrics = new List(); + ILogger quietLogger = !benchmarkCase.Config.Options.IsSet(ConfigOptions.QuietMode) ? logger : NullLogger.Instance; - logger.WriteLineInfo("// *** Execute ***"); + quietLogger.WriteLineInfo("// *** Execute ***"); bool analyzeRunToRunVariance = benchmarkCase.Job.ResolveValue(AccuracyMode.AnalyzeLaunchVarianceCharacteristic, resolver); bool autoLaunchCount = !benchmarkCase.Job.HasValue(RunMode.LaunchCountCharacteristic); int defaultValue = analyzeRunToRunVariance ? 2 : 1; @@ -411,7 +414,7 @@ private static (bool success, List executeResults, List m string printedLaunchCount = analyzeRunToRunVariance && autoLaunchCount && launchIndex <= 2 ? "" : " / " + launchCount; - logger.WriteLineInfo($"// Launch: {launchIndex}{printedLaunchCount}"); + quietLogger.WriteLineInfo($"// Launch: {launchIndex}{printedLaunchCount}"); // use diagnoser only for the last run (we need single result, not many) bool useDiagnoser = launchIndex == launchCount && noOverheadCompositeDiagnoser != null; @@ -489,12 +492,14 @@ private static (bool success, List executeResults, List m private static ExecuteResult RunExecute(ILogger logger, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, IToolchain toolchain, BuildResult buildResult, IResolver resolver, IDiagnoser diagnoser, int launchIndex) { + ILogger quietLogger = !benchmarkCase.Config.Options.IsSet(ConfigOptions.QuietMode) ? logger : NullLogger.Instance; + var executeResult = toolchain.Executor.Execute( new ExecuteParameters( buildResult, benchmarkCase, benchmarkId, - logger, + quietLogger, resolver, launchIndex, diagnoser)); From c70dc82582d04b508b76f8b152c8a93fb36d8dcd Mon Sep 17 00:00:00 2001 From: Francisco Loureiro Date: Tue, 18 Oct 2022 10:21:07 +0000 Subject: [PATCH 3/4] add quiet log to statistics and process finish messages --- .../Running/BenchmarkRunnerClean.cs | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index dd299c1f75..ba6fc28671 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -144,14 +144,13 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, var cultureInfo = config.CultureInfo ?? DefaultCultureInfo.Instance; var reports = new List(); string title = GetTitle(new[] { benchmarkRunInfo }); + var quietLogger = GetQuietLogger(config, logger); logger.WriteLineInfo($"// Found {benchmarks.Length} benchmarks:"); foreach (var benchmark in benchmarks) logger.WriteLineInfo($"// {benchmark.DisplayInfo}"); logger.WriteLine(); - ILogger quietLogger = !config.Options.IsSet(ConfigOptions.QuietMode) ? logger : NullLogger.Instance; - using (var powerManagementApplier = new PowerManagementApplier(quietLogger)) { bool stop = false; @@ -179,7 +178,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, { var statistics = report.GetResultRuns().GetStatistics(); var formatter = statistics.CreateNanosecondFormatter(cultureInfo); - logger.WriteLineStatistic(statistics.ToString(cultureInfo, formatter)); + quietLogger.WriteLineStatistic(statistics.ToString(cultureInfo, formatter)); } if (!report.Success && config.Options.IsSet(ConfigOptions.StopOnFirstError)) @@ -215,7 +214,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, logger.WriteLine(); benchmarksToRunCount -= stop ? benchmarks.Length - i : 1; - LogProgress(benchmarksToRunCount == 0 ? logger : quietLogger, in runsChronometer, totalBenchmarkCount, benchmarksToRunCount); + LogProgress(logger, in runsChronometer, totalBenchmarkCount, benchmarksToRunCount); } } @@ -397,7 +396,7 @@ private static (bool success, List executeResults, List m { var executeResults = new List(); var metrics = new List(); - ILogger quietLogger = !benchmarkCase.Config.Options.IsSet(ConfigOptions.QuietMode) ? logger : NullLogger.Instance; + var quietLogger = GetQuietLogger(benchmarkCase.Config, logger); quietLogger.WriteLineInfo("// *** Execute ***"); bool analyzeRunToRunVariance = benchmarkCase.Job.ResolveValue(AccuracyMode.AnalyzeLaunchVarianceCharacteristic, resolver); @@ -492,8 +491,7 @@ private static (bool success, List executeResults, List m private static ExecuteResult RunExecute(ILogger logger, BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, IToolchain toolchain, BuildResult buildResult, IResolver resolver, IDiagnoser diagnoser, int launchIndex) { - ILogger quietLogger = !benchmarkCase.Config.Options.IsSet(ConfigOptions.QuietMode) ? logger : NullLogger.Instance; - + var quietLogger = GetQuietLogger(benchmarkCase.Config, logger); var executeResult = toolchain.Executor.Execute( new ExecuteParameters( buildResult, @@ -513,11 +511,11 @@ private static ExecuteResult RunExecute(ILogger logger, BenchmarkCase benchmarkC { if (executeResult.ExitCode is int exitCode) { - logger.WriteLineInfo($"// Benchmark Process {executeResult.ProcessId} has exited with code {exitCode}."); + quietLogger.WriteLineInfo($"// Benchmark Process {executeResult.ProcessId} has exited with code {exitCode}."); } else { - logger.WriteLineInfo($"// Benchmark Process {executeResult.ProcessId} failed to exit."); + quietLogger.WriteLineInfo($"// Benchmark Process {executeResult.ProcessId} failed to exit."); } } @@ -629,5 +627,9 @@ private static void LogProgress(ILogger logger, in StartedClock runsChronometer, $" Estimated finish {estimatedEnd:yyyy-MM-dd H:mm} ({(int)fromNow.TotalHours}h {fromNow.Minutes}m from now) **"; logger.WriteLineHeader(message); } + + private static ILogger GetQuietLogger(ImmutableConfig config, ILogger logger) + => !config.Options.IsSet(ConfigOptions.QuietMode) ? logger : NullLogger.Instance; + } } From 8a575488e5e77a51745449e70ca4ad4fc6ba42ea Mon Sep 17 00:00:00 2001 From: Francisco Loureiro Date: Tue, 18 Oct 2022 11:59:30 +0000 Subject: [PATCH 4/4] fix docs --- src/BenchmarkDotNet/Attributes/QuietModeAttribute.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Attributes/QuietModeAttribute.cs b/src/BenchmarkDotNet/Attributes/QuietModeAttribute.cs index c40e890c20..ab2e7da23a 100644 --- a/src/BenchmarkDotNet/Attributes/QuietModeAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/QuietModeAttribute.cs @@ -5,7 +5,7 @@ namespace BenchmarkDotNet.Attributes { /// - /// determines if all auto-generated files should be kept or removed after running the benchmarks + /// Run benchmars in quiet mode. /// [PublicAPI] [AttributeUsage(AttributeTargets.Class)]