Skip to content

Commit 356a395

Browse files
adamsitnikadamsitnik
committed
move BenchmarkRunner's core to separate class to .Core project so it can be reused by others, #225 
1 parent 19426ae commit 356a395

File tree

4 files changed

+314
-300
lines changed

4 files changed

+314
-300
lines changed

‎src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs

Lines changed: 303 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,303 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using BenchmarkDotNet.Configs;
6+
using BenchmarkDotNet.Exporters;
7+
using BenchmarkDotNet.Extensions;
8+
using BenchmarkDotNet.Helpers;
9+
using BenchmarkDotNet.Horology;
10+
using BenchmarkDotNet.Jobs;
11+
using BenchmarkDotNet.Loggers;
12+
using BenchmarkDotNet.Mathematics;
13+
using BenchmarkDotNet.Reports;
14+
using BenchmarkDotNet.Toolchains;
15+
using BenchmarkDotNet.Toolchains.Results;
16+
using BenchmarkDotNet.Validators;
17+
18+
namespace BenchmarkDotNet.Running
19+
{
20+
internal static class BenchmarkRunnerCore
21+
{
22+
private static int benchmarkRunIndex;
23+
24+
internal static Summary Run(Benchmark[] benchmarks, IConfig config, Func<IJob, IToolchain> toolchainProvider)
25+
{
26+
config = BenchmarkConverter.GetFullConfig(benchmarks.FirstOrDefault()?.Target.Type, config);
27+
28+
var title = GetTitle(benchmarks);
29+
var rootArtifactsFolderPath = GetRootArtifactsFolderPath();
30+
31+
using (var logStreamWriter = Portability.StreamWriter.FromPath(Path.Combine(rootArtifactsFolderPath, title + ".log")))
32+
{
33+
var logger = new CompositeLogger(config.GetCompositeLogger(), new StreamLogger(logStreamWriter));
34+
benchmarks = GetSupportedBenchmarks(benchmarks, logger, toolchainProvider);
35+
36+
var summary = Run(benchmarks, logger, title, config, rootArtifactsFolderPath, toolchainProvider);
37+
if (!summary.HasCriticalValidationErrors)
38+
{
39+
config.GetCompositeExporter().ExportToFiles(summary).ToArray();
40+
}
41+
return summary;
42+
}
43+
}
44+
45+
private static string GetTitle(IList<Benchmark> benchmarks)
46+
{
47+
var types = benchmarks.Select(b => b.Target.Type.Name).Distinct().ToArray();
48+
if (types.Length == 1)
49+
return types[0];
50+
benchmarkRunIndex++;
51+
return $"BenchmarkRun-{benchmarkRunIndex:##000}-{DateTime.Now:yyyy-MM-dd-hh-mm-ss}";
52+
}
53+
54+
private static Summary Run(Benchmark[] benchmarks, ILogger logger, string title, IConfig config, string rootArtifactsFolderPath, Func<IJob, IToolchain> toolchainProvider)
55+
{
56+
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
57+
logger.WriteLineInfo("// Found benchmarks:");
58+
foreach (var benchmark in benchmarks)
59+
logger.WriteLineInfo($"// {benchmark.ShortInfo}");
60+
logger.WriteLine();
61+
62+
var validationErrors = Validate(benchmarks, logger, config);
63+
if (validationErrors.Any(validationError => validationError.IsCritical))
64+
{
65+
return Summary.CreateFailed(benchmarks, title, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), validationErrors);
66+
}
67+
68+
var globalChronometer = Chronometer.Start();
69+
var reports = new List<BenchmarkReport>();
70+
foreach (var benchmark in benchmarks)
71+
{
72+
var report = Run(benchmark, logger, config, rootArtifactsFolderPath, toolchainProvider);
73+
reports.Add(report);
74+
if (report.GetResultRuns().Any())
75+
logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
76+
77+
logger.WriteLine();
78+
}
79+
var clockSpan = globalChronometer.Stop();
80+
81+
var summary = new Summary(title, reports, HostEnvironmentInfo.GetCurrent(), config, GetResultsFolderPath(rootArtifactsFolderPath), clockSpan.GetTimeSpan(), validationErrors);
82+
83+
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
84+
logger.WriteLine();
85+
86+
logger.WriteLineHeader("// * Export *");
87+
var currentDirectory = Directory.GetCurrentDirectory();
88+
foreach (var file in config.GetCompositeExporter().ExportToFiles(summary))
89+
{
90+
logger.WriteLineInfo($" {file.Replace(currentDirectory, string.Empty).Trim('/', '\\')}");
91+
}
92+
logger.WriteLine();
93+
94+
logger.WriteLineHeader("// * Detailed results *");
95+
96+
// TODO: make exporter
97+
foreach (var report in reports)
98+
{
99+
logger.WriteLineInfo(report.Benchmark.ShortInfo);
100+
logger.WriteLineStatistic(report.GetResultRuns().GetStatistics().ToTimeStr());
101+
logger.WriteLine();
102+
}
103+
104+
LogTotalTime(logger, clockSpan.GetTimeSpan());
105+
logger.WriteLine();
106+
107+
logger.WriteLineHeader("// * Summary *");
108+
MarkdownExporter.Console.ExportToLog(summary, logger);
109+
110+
// TODO: make exporter
111+
var warnings = config.GetCompositeAnalyser().Analyse(summary).ToList();
112+
if (warnings.Count > 0)
113+
{
114+
logger.WriteLine();
115+
logger.WriteLineError("// * Warnings * ");
116+
foreach (var warning in warnings)
117+
logger.WriteLineError($"{warning.Message}");
118+
}
119+
120+
if (config.GetDiagnosers().Count() > 0)
121+
{
122+
logger.WriteLine();
123+
config.GetCompositeDiagnoser().DisplayResults(logger);
124+
}
125+
126+
logger.WriteLine();
127+
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
128+
return summary;
129+
}
130+
131+
private static ValidationError[] Validate(IList<Benchmark> benchmarks, ILogger logger, IConfig config)
132+
{
133+
logger.WriteLineInfo("// Validating benchmarks:");
134+
var validationErrors = config.GetCompositeValidator().Validate(benchmarks).ToArray();
135+
foreach (var validationError in validationErrors)
136+
{
137+
logger.WriteLineError(validationError.Message);
138+
}
139+
return validationErrors;
140+
}
141+
142+
internal static void LogTotalTime(ILogger logger, TimeSpan time, string message = "Total time")
143+
{
144+
var hhMmSs = $"{time.TotalHours:00}:{time:mm\\:ss}";
145+
var totalSecs = $"{time.TotalSeconds.ToStr()} sec";
146+
logger.WriteLineStatistic($"{message}: {hhMmSs} ({totalSecs})");
147+
}
148+
149+
private static BenchmarkReport Run(Benchmark benchmark, ILogger logger, IConfig config, string rootArtifactsFolderPath, Func<IJob, IToolchain> toolchainProvider)
150+
{
151+
var toolchain = toolchainProvider(benchmark.Job);
152+
153+
logger.WriteLineHeader("// **************************");
154+
logger.WriteLineHeader("// Benchmark: " + benchmark.ShortInfo);
155+
156+
var generateResult = Generate(logger, toolchain, benchmark, rootArtifactsFolderPath, config);
157+
158+
try
159+
{
160+
if (!generateResult.IsGenerateSuccess)
161+
return new BenchmarkReport(benchmark, generateResult, null, null, null);
162+
163+
var buildResult = Build(logger, toolchain, generateResult, benchmark);
164+
if (!buildResult.IsBuildSuccess)
165+
return new BenchmarkReport(benchmark, generateResult, buildResult, null, null);
166+
167+
List<ExecuteResult> executeResults = Execute(logger, benchmark, toolchain, buildResult, config);
168+
169+
var runs = new List<Measurement>();
170+
for (int index = 0; index < executeResults.Count; index++)
171+
{
172+
var executeResult = executeResults[index];
173+
runs.AddRange(executeResult.Data.Select(line => Measurement.Parse(logger, line, index + 1)).Where(r => r.IterationMode != IterationMode.Unknown));
174+
}
175+
176+
return new BenchmarkReport(benchmark, generateResult, buildResult, executeResults, runs);
177+
}
178+
finally
179+
{
180+
if (!config.KeepBenchmarkFiles)
181+
{
182+
generateResult.ArtifactsPaths?.RemoveBenchmarkFiles();
183+
}
184+
}
185+
}
186+
187+
private static GenerateResult Generate(ILogger logger, IToolchain toolchain, Benchmark benchmark, string rootArtifactsFolderPath, IConfig config)
188+
{
189+
logger.WriteLineInfo("// *** Generate *** ");
190+
var generateResult = toolchain.Generator.GenerateProject(benchmark, logger, rootArtifactsFolderPath, config);
191+
if (generateResult.IsGenerateSuccess)
192+
{
193+
logger.WriteLineInfo("// Result = Success");
194+
logger.WriteLineInfo($"// {nameof(generateResult.ArtifactsPaths.BinariesDirectoryPath)} = {generateResult.ArtifactsPaths?.BinariesDirectoryPath}");
195+
}
196+
else
197+
{
198+
logger.WriteLineError("// Result = Failure");
199+
if (generateResult.GenerateException != null)
200+
logger.WriteLineError($"// Exception: {generateResult.GenerateException.Message}");
201+
}
202+
logger.WriteLine();
203+
return generateResult;
204+
}
205+
206+
private static BuildResult Build(ILogger logger, IToolchain toolchain, GenerateResult generateResult, Benchmark benchmark)
207+
{
208+
logger.WriteLineInfo("// *** Build ***");
209+
var buildResult = toolchain.Builder.Build(generateResult, logger, benchmark);
210+
if (buildResult.IsBuildSuccess)
211+
{
212+
logger.WriteLineInfo("// Result = Success");
213+
}
214+
else
215+
{
216+
logger.WriteLineError("// Result = Failure");
217+
if (buildResult.BuildException != null)
218+
logger.WriteLineError($"// Exception: {buildResult.BuildException.Message}");
219+
}
220+
logger.WriteLine();
221+
return buildResult;
222+
}
223+
224+
private static List<ExecuteResult> Execute(ILogger logger, Benchmark benchmark, IToolchain toolchain, BuildResult buildResult, IConfig config)
225+
{
226+
var executeResults = new List<ExecuteResult>();
227+
228+
logger.WriteLineInfo("// *** Execute ***");
229+
var launchCount = Math.Max(1, benchmark.Job.LaunchCount.IsAuto ? 2 : benchmark.Job.LaunchCount.Value);
230+
231+
for (int processNumber = 0; processNumber < launchCount; processNumber++)
232+
{
233+
var printedProcessNumber = (benchmark.Job.LaunchCount.IsAuto && processNumber < 2) ? "" : " / " + launchCount.ToString();
234+
logger.WriteLineInfo($"// Launch: {processNumber + 1}{printedProcessNumber}");
235+
236+
var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger);
237+
238+
if (!executeResult.FoundExecutable)
239+
logger.WriteLineError("Executable not found");
240+
executeResults.Add(executeResult);
241+
242+
var measurements = executeResults
243+
.SelectMany(r => r.Data)
244+
.Select(line => Measurement.Parse(logger, line, 0))
245+
.Where(r => r.IterationMode != IterationMode.Unknown).
246+
ToArray();
247+
248+
if (!measurements.Any())
249+
{
250+
// Something went wrong during the benchmark, don't bother doing more runs
251+
logger.WriteLineError($"No more Benchmark runs will be launched as NO measurements were obtained from the previous run!");
252+
break;
253+
}
254+
255+
if (benchmark.Job.LaunchCount.IsAuto && processNumber == 1)
256+
{
257+
var idleApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.IdleTarget).Select(m => m.Nanoseconds)).Median;
258+
var mainApprox = new Statistics(measurements.Where(m => m.IterationMode == IterationMode.MainTarget).Select(m => m.Nanoseconds)).Median;
259+
var percent = idleApprox / mainApprox * 100;
260+
launchCount = (int)Math.Round(Math.Max(2, 2 + (percent - 1) / 3)); // an empirical formula
261+
}
262+
}
263+
logger.WriteLine();
264+
265+
// Do a "Diagnostic" run, but DISCARD the results, so that the overhead of Diagnostics doesn't skew the overall results
266+
if (config.GetDiagnosers().Count() > 0)
267+
{
268+
logger.WriteLineInfo($"// Run, Diagnostic");
269+
config.GetCompositeDiagnoser().Start(benchmark);
270+
var executeResult = toolchain.Executor.Execute(buildResult, benchmark, logger, config.GetCompositeDiagnoser());
271+
var allRuns = executeResult.Data.Select(line => Measurement.Parse(logger, line, 0)).Where(r => r.IterationMode != IterationMode.Unknown).ToList();
272+
var report = new BenchmarkReport(benchmark, null, null, new[] { executeResult }, allRuns);
273+
config.GetCompositeDiagnoser().Stop(benchmark, report);
274+
275+
if (!executeResult.FoundExecutable)
276+
logger.WriteLineError("Executable not found");
277+
logger.WriteLine();
278+
}
279+
280+
return executeResults;
281+
}
282+
283+
private static Benchmark[] GetSupportedBenchmarks(IList<Benchmark> benchmarks, CompositeLogger logger, Func<IJob, IToolchain> toolchainProvider)
284+
{
285+
return benchmarks.Where(benchmark => toolchainProvider(benchmark.Job).IsSupported(benchmark, logger)).ToArray();
286+
}
287+
288+
private static string GetRootArtifactsFolderPath() => CombineAndCreate(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Artifacts");
289+
290+
private static string GetResultsFolderPath(string rootArtifactsFolderPath) => CombineAndCreate(rootArtifactsFolderPath, "results");
291+
292+
private static string CombineAndCreate(string rootFolderPath, string childFolderName)
293+
{
294+
var path = Path.Combine(rootFolderPath, childFolderName);
295+
if (!Directory.Exists(path))
296+
{
297+
Directory.CreateDirectory(path);
298+
}
299+
300+
return path;
301+
}
302+
}
303+
}

0 commit comments

Comments
 (0)