Skip to content

Commit 33fe169

Browse files
filzrevfilzrev
committed
chore: add configparser related tests
1 parent b96afb0 commit 33fe169

2 files changed

Lines changed: 338 additions & 9 deletions

File tree

‎tests/BenchmarkDotNet.Tests/CommandLineOptionsTests.cs‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
using AwesomeAssertions;
2+
using BenchmarkDotNet.ConsoleArguments;
3+
using BenchmarkDotNet.Extensions;
4+
using CommandLine;
5+
using System.Reflection;
6+
7+
namespace BenchmarkDotNet.Tests;
8+
9+
public class CommandLineOptionsTests
10+
{
11+
private sealed record OptionProperty(PropertyInfo Property, OptionAttribute Option);
12+
13+
/// <summary>
14+
/// Validates that the hardcoded dictionaries in CommandLineOptions (CanonicalNames, MultiInstanceOptionNames)
15+
/// are consistent with the actual option definitions in the CommandLineOptions class.
16+
/// </summary>
17+
[Fact]
18+
public void ValidateCommandLineOptionDefinitions()
19+
{
20+
// Arrange
21+
// Collect all option definitions of CommandLineOptions via reflection.
22+
var optionProperties = typeof(CommandLineOptions)
23+
.GetProperties(BindingFlags.Public | BindingFlags.Instance)
24+
.Select(property => (Property: property, Option: property.GetCustomAttribute<OptionAttribute>()))
25+
.Where(entry => entry.Option is not null)
26+
.Select(x => new OptionProperty(x.Property, x.Option!))
27+
.ToArray();
28+
29+
// Gets all long names.
30+
var longNames = optionProperties
31+
.Select(entry => entry.Option.LongName)
32+
.ToHashSet(StringComparer.OrdinalIgnoreCase);
33+
34+
// Gets all names (long and short)
35+
var allNames = longNames.ToHashSet(StringComparer.OrdinalIgnoreCase);
36+
foreach (var option in optionProperties.Where(x => x.Option.ShortName.IsNotBlank()))
37+
allNames.Add(option.Option.ShortName);
38+
39+
// Act
40+
var canonicalNames = CommandLineOptions.CanonicalNames;
41+
var multiInstanceOptionNames = CommandLineOptions.MultiValueOptionNames;
42+
43+
// Assert
44+
// Validate CanonicalNames definitions.
45+
canonicalNames.Keys.Should().BeEquivalentTo(allNames);
46+
canonicalNames.Values.Distinct().Should().BeEquivalentTo(longNames);
47+
48+
// Validate each short/long name resolves to its own canonical long name
49+
// (Without this test, a swapped alias would pass the set checks above).
50+
var expectedCanonicals = optionProperties
51+
.SelectMany(entry => new[]
52+
{
53+
(Name: entry.Option!.LongName, Canonical: entry.Option.LongName),
54+
(Name: entry.Option.ShortName, Canonical: entry.Option.LongName)
55+
})
56+
.Where(entry => entry.Name.IsNotBlank())
57+
.ToArray();
58+
59+
foreach (var (name, expected) in expectedCanonicals)
60+
{
61+
bool isSuccess = canonicalNames.TryGetValue(name, out string? actual);
62+
isSuccess.Should().BeTrue($"Name '{name}' should be resolve from CanonicalNames.");
63+
actual.Should().Be(expected, $"Name '{name}' should resolve to '{expected}'.");
64+
}
65+
66+
// Validate MultiInstanceOptionNames definitions are consistent with the property types.
67+
foreach (var (property, option) in optionProperties)
68+
{
69+
bool isMultiValue = property.PropertyType != typeof(string) && typeof(System.Collections.IEnumerable).IsAssignableFrom(property.PropertyType);
70+
if (isMultiValue)
71+
multiInstanceOptionNames.Should().Contain(option.LongName);
72+
else
73+
multiInstanceOptionNames.Should().NotContain(option.LongName);
74+
}
75+
}
76+
}

‎tests/BenchmarkDotNet.Tests/ConfigParserTests.cs‎

Lines changed: 262 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,30 +6,30 @@
66
using BenchmarkDotNet.Engines;
77
using BenchmarkDotNet.Environments;
88
using BenchmarkDotNet.Exporters;
9-
using BenchmarkDotNet.Helpers;
10-
using BenchmarkDotNet.Reports;
119
using BenchmarkDotNet.Exporters.Csv;
1210
using BenchmarkDotNet.Exporters.Json;
1311
using BenchmarkDotNet.Exporters.OpenMetrics;
1412
using BenchmarkDotNet.Exporters.Xml;
13+
using BenchmarkDotNet.Helpers;
1514
using BenchmarkDotNet.Jobs;
1615
using BenchmarkDotNet.Loggers;
1716
using BenchmarkDotNet.Portability;
17+
using BenchmarkDotNet.Reports;
1818
using BenchmarkDotNet.Tests.Loggers;
1919
using BenchmarkDotNet.Tests.Mocks;
2020
using BenchmarkDotNet.Tests.XUnit;
2121
using BenchmarkDotNet.Toolchains;
2222
using BenchmarkDotNet.Toolchains.CoreRun;
2323
using BenchmarkDotNet.Toolchains.DotNetCli;
24+
using BenchmarkDotNet.Toolchains.Framework;
2425
using BenchmarkDotNet.Toolchains.InProcess.Emit;
2526
using BenchmarkDotNet.Toolchains.Mono;
26-
using BenchmarkDotNet.Toolchains.Wasm;
2727
using BenchmarkDotNet.Toolchains.NativeAot;
28-
using BenchmarkDotNet.Toolchains.Framework;
29-
using Perfolizer.Horology;
30-
using System.Reflection;
3128
using BenchmarkDotNet.Toolchains.NetCoreApp;
3229
using BenchmarkDotNet.Toolchains.R2R;
30+
using BenchmarkDotNet.Toolchains.Wasm;
31+
using Perfolizer.Horology;
32+
using System.Reflection;
3333

3434
namespace BenchmarkDotNet.Tests
3535
{
@@ -1025,6 +1025,7 @@ public void UserCanSpecifyWasmMainJsTemplate()
10251025
[InlineData("--filter abc", "--filter *")]
10261026
[InlineData("-f abc", "--filter *")]
10271027
[InlineData("-f *", "--filter *")]
1028+
[InlineData("--filter abc -f abc", "--filter *")]
10281029
[InlineData("--runtimes net7.0 --join", "--filter * --join --runtimes net7.0")]
10291030
[InlineData("--join abc", "--filter * --join")]
10301031
public void CheckUpdateValidArgs(string strArgs, string expected)
@@ -1036,21 +1037,273 @@ public void CheckUpdateValidArgs(string strArgs, string expected)
10361037
}
10371038

10381039
[Theory]
1039-
[InlineData("--filter abc -f abc")]
10401040
[InlineData("--runtimes net")]
1041+
[InlineData("--job dry --job short")]
1042+
[InlineData("--join --join")]
10411043
public void CheckUpdateInvalidArgs(string strArgs)
10421044
{
1045+
// Arrange
10431046
var args = strArgs.Split();
1047+
1048+
// Act
10441049
bool isSuccess = ConfigParser.TryUpdateArgs(args, out var updatedArgs, options => options.Filters = ["*"]);
10451050

1046-
Assert.Null(updatedArgs);
1047-
Assert.False(isSuccess);
1051+
// Assert
1052+
updatedArgs.Should().BeNull();
1053+
isSuccess.Should().BeFalse();
10481054
}
10491055

10501056
private string GetDummyWasmEngine()
10511057
{
10521058
// We know, that this file exists, that's enough.
10531059
return $"--wasmEngine={Assembly.GetExecutingAssembly().Location}";
10541060
}
1061+
1062+
[Fact]
1063+
public void UserCanSpecifyRuntimes_WithMultipleOptions()
1064+
{
1065+
// Arrange
1066+
var logger = new OutputLogger(Output);
1067+
string[] args = ["--runtimes", "net8.0", "--runtimes", "net9.0"];
1068+
1069+
// Act
1070+
var (isSuccess, config, options) = ConfigParser.Parse(args, logger);
1071+
1072+
// Assert
1073+
isSuccess.Should().BeTrue();
1074+
config.Should().NotBeNull();
1075+
options.Should().NotBeNull();
1076+
options.Runtimes.Should().Equal("net8.0", "net9.0");
1077+
config.GetJobs().Should().HaveCount(2);
1078+
config.GetJobs().First().Meta.Baseline.Should().BeTrue();
1079+
logger.GetLog().Should().BeEmpty();
1080+
}
1081+
1082+
[Fact]
1083+
public void UserCanSpecifyExporters_WithMultipleOptions()
1084+
{
1085+
// Arrange
1086+
var logger = new OutputLogger(Output);
1087+
string[] args = ["--exporters", "json", "--exporters", "html"];
1088+
1089+
// Act
1090+
var (isSuccess, config, _) = ConfigParser.Parse(args, logger);
1091+
1092+
// Assert
1093+
isSuccess.Should().BeTrue();
1094+
config.Should().NotBeNull();
1095+
config.GetExporters().Should().Contain(JsonExporter.Default);
1096+
config.GetExporters().Should().Contain(HtmlExporter.Default);
1097+
logger.GetLog().Should().BeEmpty();
1098+
}
1099+
1100+
[Theory]
1101+
[InlineData("--filter", "A", "--filter", "B")]
1102+
[InlineData("-f", "A", "--filter", "B")]
1103+
public void UserCanSpecifyFilter_WithMultipleOptions(params string[] args)
1104+
{
1105+
// Arrange
1106+
var logger = new OutputLogger(Output);
1107+
1108+
// Act
1109+
var (isSuccess, _, options) = ConfigParser.Parse(args, logger);
1110+
1111+
// Assert
1112+
isSuccess.Should().BeTrue();
1113+
options.Should().NotBeNull();
1114+
options.Filters.Should().Equal("A", "B");
1115+
logger.GetLog().Should().BeEmpty();
1116+
}
1117+
1118+
[Fact]
1119+
public void UserCanSpecifyCoreRunPaths_WithMultipleOptions()
1120+
{
1121+
// Arrange
1122+
var logger = new OutputLogger(Output);
1123+
var path1 = typeof(object).Assembly.Location;
1124+
var path2 = typeof(ConfigParserTests).Assembly.Location;
1125+
1126+
// Act
1127+
var (isSuccess, config, _) = ConfigParser.Parse(["--coreRun", path1, "--coreRun", path2], logger);
1128+
1129+
// Assert
1130+
isSuccess.Should().BeTrue();
1131+
config.Should().NotBeNull();
1132+
config.GetJobs().Should().HaveCount(2);
1133+
logger.GetLog().Should().BeEmpty();
1134+
}
1135+
1136+
[Fact]
1137+
public void UserCanSpecifyEnvVars_WithMultipleOptions()
1138+
{
1139+
// Arrange
1140+
var logger = new OutputLogger(Output);
1141+
string[] args = ["--envVars", "K1:V1", "--envVars", "K2:V2"];
1142+
1143+
// Act
1144+
var (isSuccess, config, _) = ConfigParser.Parse(args, logger);
1145+
1146+
// Assert
1147+
isSuccess.Should().BeTrue();
1148+
config.Should().NotBeNull();
1149+
config.GetJobs().Single().Environment.EnvironmentVariables.Should().HaveCount(2);
1150+
logger.GetLog().Should().BeEmpty();
1151+
}
1152+
1153+
[Fact]
1154+
public void UserCanSpecifyRuntimes_WithSingleOption()
1155+
{
1156+
// Arrange
1157+
var logger = new OutputLogger(Output);
1158+
string[] args = ["--runtimes", "net8.0", "net9.0"];
1159+
1160+
// Act
1161+
var (isSuccess, config, options) = ConfigParser.Parse(args, logger);
1162+
1163+
// Assert
1164+
isSuccess.Should().BeTrue();
1165+
config.Should().NotBeNull();
1166+
options.Should().NotBeNull();
1167+
config.GetJobs().Should().HaveCount(2);
1168+
logger.GetLog().Should().BeEmpty();
1169+
}
1170+
1171+
[Fact]
1172+
public void UserCanSpecifyCounters_WithMultipleOptions()
1173+
{
1174+
// Arrange
1175+
var logger = new OutputLogger(Output);
1176+
string[] args =
1177+
[
1178+
"--counters", $"{nameof(HardwareCounter.CacheMisses)}+{nameof(HardwareCounter.InstructionRetired)}",
1179+
"--counters", nameof(HardwareCounter.BranchMispredictions)
1180+
];
1181+
1182+
// Act
1183+
var (isSuccess, config, _) = ConfigParser.Parse(args, logger);
1184+
1185+
// Assert
1186+
isSuccess.Should().BeTrue();
1187+
config.Should().NotBeNull();
1188+
config!.GetHardwareCounters().Should().HaveCount(3);
1189+
logger.GetLog().Should().BeEmpty();
1190+
}
1191+
1192+
[Fact]
1193+
public void OptionAfterDashDashIsTreatedAsValue()
1194+
{
1195+
// Arrange
1196+
var logger = new OutputLogger(Output);
1197+
string[] args = ["--job", "dry", "--", "--job", "short"];
1198+
1199+
// Act
1200+
var (isSuccess, _, _) = ConfigParser.Parse(args, logger);
1201+
1202+
// Assert
1203+
isSuccess.Should().BeTrue();
1204+
logger.GetLog().Should().BeEmpty();
1205+
}
1206+
1207+
[Fact]
1208+
public void UserCanNotSpecifyCounters_MoreThan3_RaiseError()
1209+
{
1210+
// Arrange
1211+
var logger = new OutputLogger(Output);
1212+
string[] args =
1213+
[
1214+
"--counters", $"{nameof(HardwareCounter.CacheMisses)}+{nameof(HardwareCounter.InstructionRetired)}",
1215+
"--counters", $"{nameof(HardwareCounter.BranchMispredictions)}+{nameof(HardwareCounter.Timer)}"
1216+
];
1217+
1218+
// Act
1219+
var (isSuccess, _, _) = ConfigParser.Parse(args, logger);
1220+
1221+
// Assert
1222+
isSuccess.Should().BeFalse();
1223+
1224+
var log = logger.GetLog().Trim();
1225+
log.Should().Be("You can't use more than 3 HardwareCounters at the same time.");
1226+
}
1227+
1228+
[Fact]
1229+
public void UserCanNotSpecifyCounters_WrongName()
1230+
{
1231+
// Arrange
1232+
var logger = new OutputLogger(Output);
1233+
string[] args = ["--counters", nameof(HardwareCounter.CacheMisses), "--counters", "WRONG_NAME"];
1234+
1235+
// Act
1236+
var (isSuccess, _, _) = ConfigParser.Parse(args, logger);
1237+
1238+
// Assert
1239+
isSuccess.Should().BeFalse();
1240+
1241+
var log = logger.GetLog().Trim();
1242+
log.Should().Be("The provided hardware counter \"WRONG_NAME\" is invalid. Available options are: NotSet+Timer+TotalIssues+BranchInstructions+CacheMisses+BranchMispredictions+TotalCycles+UnhaltedCoreCycles+InstructionRetired+UnhaltedReferenceCycles+LlcReference+LlcMisses+BranchInstructionRetired+BranchMispredictsRetired.");
1243+
}
1244+
1245+
[Theory]
1246+
[InlineData("--join", "--join")]
1247+
[InlineData("--inProcess", "--inProcess")]
1248+
[InlineData("--memory", "--memory")]
1249+
[InlineData("-tm", "-m")]
1250+
[InlineData("-mm")]
1251+
public void UserCanNotSpecify_MultipleSameBooleanOptions(params string[] args)
1252+
{
1253+
// Arrange
1254+
var logger = new OutputLogger(Output);
1255+
1256+
// Act
1257+
var (isSuccess, _, _) = ConfigParser.Parse(args, logger);
1258+
1259+
// Assert
1260+
isSuccess.Should().BeFalse();
1261+
1262+
var log = logger.GetLog().Trim();
1263+
log.Should().Match($"Option '*' is defined multiple times.");
1264+
}
1265+
1266+
[Theory]
1267+
[InlineData("-tm")]
1268+
[InlineData("-m", "-t")]
1269+
public void UserCanSpecify_BundledDifferentBooleanOptions(params string[] args)
1270+
{
1271+
// Arrange
1272+
var logger = new OutputLogger(Output);
1273+
1274+
// Act
1275+
var (isSuccess, _, options) = ConfigParser.Parse(args, logger);
1276+
1277+
// Assert
1278+
isSuccess.Should().BeTrue();
1279+
logger.GetLog().Should().BeEmpty();
1280+
1281+
options.Should().NotBeNull();
1282+
options.UseThreadingDiagnoser.Should().BeTrue();
1283+
options.UseMemoryDiagnoser.Should().BeTrue();
1284+
}
1285+
1286+
[Theory]
1287+
[InlineData("--job", "dry", "--job", "short")]
1288+
[InlineData("--job=dry", "--job=short")]
1289+
[InlineData("-j", "dry", "-j", "short")]
1290+
[InlineData("-j", "dry", "--job", "short")]
1291+
[InlineData("--JOB", "dry", "--job", "short")]
1292+
[InlineData("--launchCount", "1", "--launchCount", "2")]
1293+
[InlineData("--title", "A", "--title", "B")]
1294+
public void UserCanNotSpecify_MultipleSameScalarOptions(params string[] args)
1295+
{
1296+
// Arrange
1297+
var logger = new OutputLogger(Output);
1298+
1299+
// Act
1300+
var (isSuccess, _, _) = ConfigParser.Parse(args, logger);
1301+
1302+
// Assert
1303+
isSuccess.Should().BeFalse();
1304+
1305+
var log = logger.GetLog().Trim();
1306+
log.Should().Match("Option '*' is defined multiple times.");
1307+
}
10551308
}
10561309
}

0 commit comments

Comments
 (0)