Skip to content

Commit d04dd1a

Browse files
timcasselltimcassell
committed
Implement SeparateLogic and fix tests.
1 parent 47904ea commit d04dd1a

File tree

11 files changed

+92
-97
lines changed

11 files changed

+92
-97
lines changed

‎src/BenchmarkDotNet/Diagnosers/CompositeDiagnoser.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ public void Handle(BenchmarkSignal signal)
8686
}
8787
}
8888

89-
if (signal != BenchmarkSignal.AfterEngine)
89+
if (signal is not (BenchmarkSignal.AfterEngine or BenchmarkSignal.SeparateLogic))
9090
{
9191
return;
9292
}

‎src/BenchmarkDotNet/Engines/InProcessSignal.cs‎ renamed to ‎src/BenchmarkDotNet/Engines/BenchmarkSignal.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,9 @@ public enum BenchmarkSignal
2424
/// after the engine has completed the run
2525
/// </summary>
2626
AfterEngine,
27+
28+
/// <summary>
29+
/// used to run some code independent of the benchmarks
30+
/// </summary>
31+
SeparateLogic,
2732
}

‎src/BenchmarkDotNet/Engines/RunResults.cs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,23 +10,23 @@
1010

1111
namespace BenchmarkDotNet.Engines
1212
{
13-
public struct RunResults
13+
public readonly struct RunResults
1414
{
1515
private readonly OutlierMode outlierMode;
1616

1717
[PublicAPI]
18-
public IReadOnlyList<Measurement> EngineMeasurements { get; }
18+
public IReadOnlyList<Measurement>? EngineMeasurements { get; }
1919

2020
[PublicAPI]
2121
public IReadOnlyList<Measurement>? Overhead
2222
=> EngineMeasurements
23-
.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual))
23+
?.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual))
2424
.ToArray();
2525

2626
[PublicAPI]
27-
public IReadOnlyList<Measurement> Workload
27+
public IReadOnlyList<Measurement>? Workload
2828
=> EngineMeasurements
29-
.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual))
29+
?.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual))
3030
.ToArray();
3131

3232
public GcStats GCStats { get; }
@@ -50,8 +50,8 @@ public RunResults(IReadOnlyList<Measurement> engineMeasurements,
5050

5151
public IEnumerable<Measurement> GetWorkloadResultMeasurements()
5252
{
53-
var overheadActualMeasurements = Overhead ?? Array.Empty<Measurement>();
54-
var workloadActualMeasurements = Workload;
53+
var overheadActualMeasurements = Overhead ?? [];
54+
var workloadActualMeasurements = Workload ?? [];
5555
if (workloadActualMeasurements.IsEmpty())
5656
yield break;
5757

@@ -78,7 +78,7 @@ public IEnumerable<Measurement> GetWorkloadResultMeasurements()
7878

7979
public IEnumerable<Measurement> GetAllMeasurements()
8080
{
81-
foreach (var measurement in EngineMeasurements)
81+
foreach (var measurement in EngineMeasurements ?? [])
8282
yield return measurement;
8383
foreach (var measurement in GetWorkloadResultMeasurements())
8484
yield return measurement;

‎src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -585,6 +585,28 @@ private static (bool success, List<ExecuteResult> executeResults, List<Metric> m
585585
logger.WriteLineInfo("// Run, Diagnostic [SeparateLogic]");
586586

587587
separateLogicCompositeDiagnoser.Handle(HostSignal.SeparateLogic, new DiagnoserActionParameters(null, benchmarkCase, benchmarkId));
588+
589+
if (compositeInProcessDiagnoser.InProcessDiagnosers.Any(d => d.GetRunMode(benchmarkCase) == Diagnosers.RunMode.SeparateLogic))
590+
{
591+
var executeResult = RunExecute(
592+
logger,
593+
benchmarkCase,
594+
benchmarkId,
595+
toolchain,
596+
buildResult,
597+
resolver,
598+
extraRunCompositeDiagnoser,
599+
compositeInProcessDiagnoser,
600+
launchCount + 1,
601+
Diagnosers.RunMode.SeparateLogic);
602+
603+
if (executeResult.IsSuccess)
604+
{
605+
metrics.AddRange(extraRunCompositeDiagnoser.ProcessResults(new DiagnoserResults(benchmarkCase, executeResult, buildResult)));
606+
}
607+
608+
logger.WriteLine();
609+
}
588610
}
589611

590612
return (true, executeResults, metrics);

‎src/BenchmarkDotNet/Templates/BenchmarkType.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@
2929
diagnoserRunMode,
3030
new BenchmarkDotNet.Diagnosers.InProcessDiagnoserActionArgs(instance)
3131
);
32+
if (diagnoserRunMode == BenchmarkDotNet.Diagnosers.RunMode.SeparateLogic)
33+
{
34+
compositeInProcessDiagnoserHandler.Handle(BenchmarkDotNet.Engines.BenchmarkSignal.SeparateLogic);
35+
return;
36+
}
3237
compositeInProcessDiagnoserHandler.Handle(BenchmarkDotNet.Engines.BenchmarkSignal.BeforeEngine);
3338

3439
BenchmarkDotNet.Engines.EngineParameters engineParameters = new BenchmarkDotNet.Engines.EngineParameters()

‎src/BenchmarkDotNet/Toolchains/InProcess/Emit/Implementation/Runnable/RunnableReuse.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public static (Job, EngineParameters, IEngineFactory) PrepareForRun<T>(T instanc
2626

2727
var errors = BenchmarkEnvironmentInfo.Validate(job);
2828
if (ValidationErrorReporter.ReportIfAny(errors, host))
29-
return (null, null, null);
29+
return default;
3030

3131
var compositeInProcessDiagnoserHandler = new CompositeInProcessDiagnoserHandler(
3232
parameters.CompositeInProcessDiagnoser.InProcessDiagnosers
@@ -42,6 +42,11 @@ public static (Job, EngineParameters, IEngineFactory) PrepareForRun<T>(T instanc
4242
parameters.DiagnoserRunMode,
4343
new InProcessDiagnoserActionArgs(instance)
4444
);
45+
if (parameters.DiagnoserRunMode == Diagnosers.RunMode.SeparateLogic)
46+
{
47+
compositeInProcessDiagnoserHandler.Handle(BenchmarkSignal.SeparateLogic);
48+
return default;
49+
}
4550
compositeInProcessDiagnoserHandler.Handle(BenchmarkSignal.BeforeEngine);
4651

4752
var engineParameters = CreateEngineParameters(instance, benchmarkCase, host, compositeInProcessDiagnoserHandler);

‎src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitRunner.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,11 @@ public static void RunCore(IHost host, ExecuteParameters parameters)
146146
parameters.DiagnoserRunMode,
147147
new Diagnosers.InProcessDiagnoserActionArgs(instance)
148148
);
149+
if (parameters.DiagnoserRunMode == Diagnosers.RunMode.SeparateLogic)
150+
{
151+
compositeInProcessDiagnoserHandler.Handle(BenchmarkSignal.SeparateLogic);
152+
return;
153+
}
149154
compositeInProcessDiagnoserHandler.Handle(BenchmarkSignal.BeforeEngine);
150155

151156
var engineParameters = new EngineParameters

‎tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs‎

Lines changed: 19 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -11,39 +11,43 @@
1111

1212
namespace BenchmarkDotNet.IntegrationTests.Diagnosers;
1313

14-
public abstract class BaseMockInProcessDiagnoser : IInProcessDiagnoser
14+
public abstract class BaseMockInProcessDiagnoser(RunMode runMode) : IInProcessDiagnoser
1515
{
16-
public static Queue<BaseMockInProcessDiagnoser> s_completedResults = new();
16+
public static Queue<string> s_completedResults = new();
1717

1818
public Dictionary<BenchmarkCase, string> Results { get; } = [];
1919

20-
public abstract string DiagnoserName { get; }
21-
public abstract RunMode DiagnoserRunMode { get; }
22-
public abstract string ExpectedResult { get; }
20+
public RunMode RunMode { get; } = runMode;
21+
public string ExpectedResult => $"MockResult-{RunMode}";
2322

24-
public IEnumerable<string> Ids => [DiagnoserName];
23+
public IEnumerable<string> Ids => [GetType().Name];
2524

2625
public IEnumerable<IExporter> Exporters => [];
2726

2827
public IEnumerable<IAnalyser> Analysers => [];
2928

30-
public void DisplayResults(ILogger logger) => logger.WriteLine($"{DiagnoserName} results: [{string.Join(", ", Results.Values)}]");
29+
public void DisplayResults(ILogger logger) => logger.WriteLine($"{GetType().Name} results: [{string.Join(", ", Results.Values)}]");
3130

32-
public RunMode GetRunMode(BenchmarkCase benchmarkCase) => DiagnoserRunMode;
31+
public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode;
3332

3433
public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
3534

3635
public IEnumerable<Metric> ProcessResults(DiagnoserResults results) => [];
3736

3837
public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => [];
3938

40-
public abstract (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase);
39+
public (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
40+
=> RunMode == RunMode.None
41+
? default
42+
: (typeof(MockInProcessDiagnoserHandler), ExpectedResult);
4143

4244
public virtual IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
4345
{
4446
var (handlerType, serializedConfig) = GetSeparateProcessHandlerTypeAndSerializedConfig(benchmarkCase);
4547
if (handlerType == null)
48+
{
4649
return null;
50+
}
4751
var handler = (IInProcessDiagnoserHandler)Activator.CreateInstance(handlerType);
4852
handler.Initialize(serializedConfig);
4953
return handler;
@@ -52,16 +56,14 @@ public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
5256
public void DeserializeResults(BenchmarkCase benchmarkCase, string results)
5357
{
5458
Results.Add(benchmarkCase, results);
55-
s_completedResults.Enqueue(this);
59+
s_completedResults.Enqueue(results);
5660
}
5761
}
5862

59-
public abstract class BaseMockInProcessDiagnoserHandler : IInProcessDiagnoserHandler
63+
public sealed class MockInProcessDiagnoserHandler : IInProcessDiagnoserHandler
6064
{
6165
private string _result;
6266

63-
protected BaseMockInProcessDiagnoserHandler() { }
64-
6567
public void Initialize(string? serializedConfig)
6668
{
6769
_result = serializedConfig ?? string.Empty;
@@ -72,61 +74,7 @@ public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs args) {
7274
public string SerializeResults() => _result;
7375
}
7476

75-
public sealed class MockInProcessDiagnoser : BaseMockInProcessDiagnoser
76-
{
77-
public override string DiagnoserName => nameof(MockInProcessDiagnoser);
78-
public override RunMode DiagnoserRunMode => RunMode.NoOverhead;
79-
public override string ExpectedResult => "MockResult";
80-
81-
public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
82-
=> (typeof(MockInProcessDiagnoserHandler), ExpectedResult);
83-
}
84-
85-
public sealed class MockInProcessDiagnoserHandler : BaseMockInProcessDiagnoserHandler
86-
{
87-
}
88-
89-
public sealed class MockInProcessDiagnoserExtraRun : BaseMockInProcessDiagnoser
90-
{
91-
public override string DiagnoserName => nameof(MockInProcessDiagnoserExtraRun);
92-
public override RunMode DiagnoserRunMode => RunMode.ExtraRun;
93-
public override string ExpectedResult => "ExtraRunResult";
94-
95-
public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
96-
=> (typeof(MockInProcessDiagnoserExtraRunHandler), ExpectedResult);
97-
}
98-
99-
public sealed class MockInProcessDiagnoserExtraRunHandler : BaseMockInProcessDiagnoserHandler
100-
{
101-
}
102-
103-
public sealed class MockInProcessDiagnoserNone : BaseMockInProcessDiagnoser
104-
{
105-
public override string DiagnoserName => nameof(MockInProcessDiagnoserNone);
106-
public override RunMode DiagnoserRunMode => RunMode.None;
107-
public override string ExpectedResult => "NoneResult";
108-
109-
public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
110-
=> default; // Returns default when RunMode is None
111-
112-
public override IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
113-
=> null; // Returns null when RunMode is None
114-
}
115-
116-
public sealed class MockInProcessDiagnoserNoneHandler : BaseMockInProcessDiagnoserHandler
117-
{
118-
}
119-
120-
public sealed class MockInProcessDiagnoserSeparateLogic : BaseMockInProcessDiagnoser
121-
{
122-
public override string DiagnoserName => nameof(MockInProcessDiagnoserSeparateLogic);
123-
public override RunMode DiagnoserRunMode => RunMode.SeparateLogic;
124-
public override string ExpectedResult => "SeparateLogicResult";
125-
126-
public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
127-
=> (typeof(MockInProcessDiagnoserSeparateLogicHandler), ExpectedResult);
128-
}
129-
130-
public sealed class MockInProcessDiagnoserSeparateLogicHandler : BaseMockInProcessDiagnoserHandler
131-
{
132-
}
77+
// Diagnosers are made unique per-type rather than per-instance, so we have to create separate types to test multiple.
78+
public sealed class MockInProcessDiagnoser1(RunMode runMode) : BaseMockInProcessDiagnoser(runMode) { }
79+
public sealed class MockInProcessDiagnoser2(RunMode runMode) : BaseMockInProcessDiagnoser(runMode) { }
80+
public sealed class MockInProcessDiagnoser3(RunMode runMode) : BaseMockInProcessDiagnoser(runMode) { }

‎tests/BenchmarkDotNet.IntegrationTests/InProcessDiagnoserTests.cs‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
using BenchmarkDotNet.IntegrationTests.Diagnosers;
99
using BenchmarkDotNet.Jobs;
1010
using BenchmarkDotNet.Tests.Loggers;
11-
using BenchmarkDotNet.Toolchains;
1211
using BenchmarkDotNet.Toolchains.InProcess.Emit;
1312
using BenchmarkDotNet.Toolchains.InProcess.NoEmit;
1413
using Xunit;
@@ -64,20 +63,25 @@ public static IEnumerable<object[]> GetTestCombinations()
6463
{
6564
foreach (var runModes in GetRunModeCombinations(count))
6665
{
66+
// Default toolchain is much slower than in-process toolchains, so to prevent CI from taking too much time, we skip combinations with duplicate run modes.
67+
if (toolchain == ToolchainType.Default && runModes.Length == 3
68+
&& (runModes[0] == runModes[1] || runModes[0] == runModes[2] || runModes[1] == runModes[2]))
69+
{
70+
continue;
71+
}
6772
yield return [runModes, toolchain];
6873
}
6974
}
7075
}
7176
}
7277

73-
private static BaseMockInProcessDiagnoser CreateDiagnoser(RunMode runMode)
74-
=> runMode switch
78+
private static BaseMockInProcessDiagnoser CreateDiagnoser(RunMode runMode, int index)
79+
=> index switch
7580
{
76-
RunMode.None => new MockInProcessDiagnoserNone(),
77-
RunMode.NoOverhead => new MockInProcessDiagnoser(),
78-
RunMode.ExtraRun => new MockInProcessDiagnoserExtraRun(),
79-
RunMode.SeparateLogic => new MockInProcessDiagnoserSeparateLogic(),
80-
_ => throw new ArgumentException($"Unsupported run mode: {runMode}")
81+
0 => new MockInProcessDiagnoser1(runMode),
82+
1 => new MockInProcessDiagnoser2(runMode),
83+
2 => new MockInProcessDiagnoser3(runMode),
84+
_ => throw new ArgumentException($"Unsupported index: {index}")
8185
};
8286

8387
private ManualConfig CreateConfig(ToolchainType toolchain)
@@ -111,7 +115,7 @@ public void MultipleInProcessDiagnosersWork(RunMode[] runModes, ToolchainType to
111115

112116
foreach (var diagnoser in diagnosers)
113117
{
114-
if (diagnoser.DiagnoserRunMode == RunMode.None)
118+
if (diagnoser.RunMode == RunMode.None)
115119
{
116120
Assert.Empty(diagnoser.Results);
117121
}
@@ -123,16 +127,17 @@ public void MultipleInProcessDiagnosersWork(RunMode[] runModes, ToolchainType to
123127
}
124128
}
125129
Assert.Equal(
126-
BaseMockInProcessDiagnoser.s_completedResults,
127130
diagnosers
128-
.Where(d => d.DiagnoserRunMode != RunMode.None)
129-
.OrderBy(d => d.DiagnoserRunMode switch
131+
.Where(d => d.RunMode != RunMode.None)
132+
.OrderBy(d => d.RunMode switch
130133
{
131134
RunMode.NoOverhead => 0,
132135
RunMode.ExtraRun => 1,
133136
RunMode.SeparateLogic => 2,
134137
_ => 3
135138
})
139+
.Select(d => d.ExpectedResult),
140+
BaseMockInProcessDiagnoser.s_completedResults
136141
);
137142
BaseMockInProcessDiagnoser.s_completedResults.Clear();
138143
}

‎tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ public void NativeAotSupportsInProcessDiagnosers()
6262
if (!GetShouldRunTest())
6363
return;
6464

65-
var diagnoser = new MockInProcessDiagnoser();
65+
var diagnoser = new MockInProcessDiagnoser1(BenchmarkDotNet.Diagnosers.RunMode.NoOverhead);
6666
var config = GetConfig().AddDiagnoser(diagnoser);
6767

6868
try
@@ -80,7 +80,7 @@ public void NativeAotSupportsInProcessDiagnosers()
8080
}
8181

8282
Assert.Equal([diagnoser.ExpectedResult], diagnoser.Results.Values);
83-
Assert.Equal([diagnoser], BaseMockInProcessDiagnoser.s_completedResults);
83+
Assert.Equal([diagnoser.ExpectedResult], BaseMockInProcessDiagnoser.s_completedResults);
8484
BaseMockInProcessDiagnoser.s_completedResults.Clear();
8585
}
8686

0 commit comments

Comments
 (0)