chore: Allow multiple CLI option parameter with same key

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -274,5 +274,127 @@ public static IEnumerable<Example> Examples
         }
 
         private static string Escape(string input) => UserInteractionHelper.EscapeCommandExample(input);
+
+        /// <summary>
+        /// All options, keyed by short and long name (case-insensitive).
+        /// Value is the canonical long name.
+        /// Keep in sync with the <c>[Option]</c> attributes above (hardcoded by design, no reflection for NativeAOT/trimming safety).
+        /// </summary>
+        internal static readonly IReadOnlyDictionary<string, string> CanonicalNames =
+            new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
+            {
+                // Aliases
+                ["j"] = "job",
+                ["r"] = "runtimes",
+                ["e"] = "exporters",
+                ["m"] = "memory",
+                ["t"] = "threading",
+                ["d"] = "disasm",
+                ["p"] = "profiler",
+                ["f"] = "filter",
+                ["h"] = "hide",
+                ["i"] = "inProcess",
+                ["a"] = "artifacts",
+
+                // Names
+                ["job"] = "job",
+                ["runtimes"] = "runtimes",
+                ["exporters"] = "exporters",
+                ["memory"] = "memory",
+                ["threading"] = "threading",
+                ["exceptions"] = "exceptions",
+                ["disasm"] = "disasm",
+                ["profiler"] = "profiler",
+                ["filter"] = "filter",
+                ["hide"] = "hide",
+                ["inProcess"] = "inProcess",
+                ["artifacts"] = "artifacts",
+                ["outliers"] = "outliers",
+                ["affinity"] = "affinity",
+                ["allStats"] = "allStats",
+                ["allCategories"] = "allCategories",
+                ["anyCategories"] = "anyCategories",
+                ["anyJobCategories"] = "anyJobCategories",
+                ["attribute"] = "attribute",
+                ["join"] = "join",
+                ["title"] = "title",
+                ["keepFiles"] = "keepFiles",
+                ["noOverwrite"] = "noOverwrite",
+                ["counters"] = "counters",
+                ["cli"] = "cli",
+                ["packages"] = "packages",
+                ["freshPackages"] = "freshPackages",
+                ["coreRun"] = "coreRun",
+                ["monoPath"] = "monoPath",
+                ["ilCompilerVersion"] = "ilCompilerVersion",
+                ["ilcPackages"] = "ilcPackages",
+                ["launchCount"] = "launchCount",
+                ["warmupCount"] = "warmupCount",
+                ["minWarmupCount"] = "minWarmupCount",
+                ["maxWarmupCount"] = "maxWarmupCount",
+                ["iterationTime"] = "iterationTime",
+                ["iterationCount"] = "iterationCount",
+                ["minIterationCount"] = "minIterationCount",
+                ["maxIterationCount"] = "maxIterationCount",
+                ["invocationCount"] = "invocationCount",
+                ["unrollFactor"] = "unrollFactor",
+                ["strategy"] = "strategy",
+                ["platform"] = "platform",
+                ["runOncePerIteration"] = "runOncePerIteration",
+                ["info"] = "info",
+                ["apples"] = "apples",
+                ["list"] = "list",
+                ["disasmDepth"] = "disasmDepth",
+                ["disasmFilter"] = "disasmFilter",
+                ["disasmDiff"] = "disasmDiff",
+                ["logBuildOutput"] = "logBuildOutput",
+                ["generateBinLog"] = "generateBinLog",
+                ["buildTimeout"] = "buildTimeout",
+                ["wakeLock"] = "wakeLock",
+                ["stopOnFirstError"] = "stopOnFirstError",
+                ["statisticalTest"] = "statisticalTest",
+                ["disableLogFile"] = "disableLogFile",
+                ["maxWidth"] = "maxWidth",
+                ["envVars"] = "envVars",
+                ["memoryRandomization"] = "memoryRandomization",
+                ["jitTieringMode"] = "jitTieringMode",
+                ["wasmEngine"] = "wasmEngine",
+                ["wasmArgs"] = "wasmArgs",
+                ["wasmMainJsTemplate"] = "wasmMainJsTemplate",
+                ["customRuntimePack"] = "customRuntimePack",
+                ["AOTCompilerPath"] = "AOTCompilerPath",
+                ["wasmProcessTimeout"] = "wasmProcessTimeout",
+                ["noForcedGCs"] = "noForcedGCs",
+                ["evaluateOverhead"] = "evaluateOverhead",
+                ["consumeTasksSynchronously"] = "consumeTasksSynchronously",
+                ["resume"] = "resume"
+            };
+
+        /// <summary>
+        /// Canonical long names of multi values options (It can be specified multiple times)
+        /// </summary>
+        internal static readonly ISet<string> MultiValueOptionNames =
+            new HashSet<string>(StringComparer.OrdinalIgnoreCase)
+            {
+                "runtimes",
+                "exporters",
+                "filter",
+                "hide",
+                "allCategories",
+                "anyCategories",
+                "anyJobCategories",
+                "attribute",
+                "counters",
+                "coreRun",
+                "envVars",
+                "disasmFilter"
+            };
+
+        /// <summary>
+        /// Canonical long names of scalar value options.
+        /// These options must appear at most once.
+        /// </summary>
+        internal static readonly ISet<string> ScalarValueOptionNames =
+            new HashSet<string>(CanonicalNames.Values.Except(MultiValueOptionNames), StringComparer.OrdinalIgnoreCase);
     }
 }

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -15,19 +15,19 @@
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Toolchains;
 using BenchmarkDotNet.Toolchains.CoreRun;
+using BenchmarkDotNet.Toolchains.Framework;
 using BenchmarkDotNet.Toolchains.Mono;
-using BenchmarkDotNet.Toolchains.Wasm;
 using BenchmarkDotNet.Toolchains.NativeAot;
-using BenchmarkDotNet.Toolchains.Framework;
+using BenchmarkDotNet.Toolchains.NetCoreApp;
 using BenchmarkDotNet.Toolchains.R2R;
+using BenchmarkDotNet.Toolchains.Wasm;
 using CommandLine;
 using Perfolizer.Horology;
 using Perfolizer.Mathematics.OutlierDetection;
 using Perfolizer.Metrology;
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
 using System.Text;
-using BenchmarkDotNet.Toolchains.NetCoreApp;
 
 namespace BenchmarkDotNet.ConsoleArguments
 {
@@ -127,6 +127,12 @@ public static (bool isSuccess, IConfig? config, CommandLineOptions? options) Par
             }
 
             args = expandedArgs;
+            if (TryFindDuplicateScalarOption(args, out string? duplicateOptionName))
+            {
+                logger.WriteLineError($"Option '--{duplicateOptionName}' is defined multiple times.");
+                return (false, default, default);
+            }
+
             using (var parser = CreateParser(logger))
             {
                 parser
@@ -244,6 +250,12 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act
             (bool isSuccess, CommandLineOptions? options) result = default;
 
             ILogger logger = NullLogger.Instance;
+            if (TryFindDuplicateScalarOption(args, out _))
+            {
+                updatedArgs = null;
+                return false;
+            }
+
             using (var parser = CreateParser(logger))
             {
                 parser
@@ -267,6 +279,7 @@ internal static bool TryUpdateArgs(string[] args, out string[]? updatedArgs, Act
         private static Parser CreateParser(ILogger logger)
             => new Parser(settings =>
             {
+                settings.AllowMultiInstance = true;
                 settings.CaseInsensitiveEnumValues = true;
                 settings.CaseSensitive = false;
                 settings.EnableDashDash = true;
@@ -275,6 +288,79 @@ private static Parser CreateParser(ILogger logger)
                 settings.MaximumDisplayWidth = Math.Max(MinimumDisplayWidth, GetMaximumDisplayWidth());
             });
 
+        /// <summary>
+        /// Detects a repeated scalar option in the (response-file expanded) args.
+        /// </summary>
+        internal static bool TryFindDuplicateScalarOption(string[] args, out string? duplicateOptionName)
+        {
+            var hashSet = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+            duplicateOptionName = null;
+
+            foreach (string arg in args)
+            {
+                if (arg.Equals("--", StringComparison.Ordinal))
+                    break;
+
+                if (arg.StartsWith("--", StringComparison.Ordinal))
+                {
+                    string text = arg.Substring(2);
+                    if (text.Length == 0)
+                        continue;
+
+                    int equalsIndex = text.IndexOf('=');
+                    string name = equalsIndex >= 0 ? text.Substring(0, equalsIndex) : text;
+                    if (name.Length == 0)
+                        continue;
+
+                    if (!TryGetScalarValueOption(name, out var canonicalName))
+                        continue;
+
+                    if (!hashSet.Add(canonicalName))
+                    {
+                        duplicateOptionName = canonicalName;
+                        return true;
+                    }
+                }
+                else if (arg.StartsWith("-", StringComparison.Ordinal) && arg.Length >= 2)
+                {
+                    // Truncate an attached value (e.g. "-j=dry" scans only "j"), mirroring the long-option branch.
+                    string shorts = arg.Substring(1);
+                    int equalsIndex = shorts.IndexOf('=');
+                    if (equalsIndex >= 0)
+                        shorts = shorts.Substring(0, equalsIndex);
+
+                    // Handle bundling of single-character options (e.g. -tm is equivalent to -t -m).
+                    // Scanning stops at the first unknown char: like the parser, the rest is treated as a value.
+                    foreach (char shortChar in shorts)
+                    {
+                        string shortName = shortChar.ToString();
+                        if (!TryGetScalarValueOption(shortName, out var canonicalName))
+                            break;
+
+                        if (!hashSet.Add(canonicalName))
+                        {
+                            duplicateOptionName = canonicalName;
+                            return true;
+                        }
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        private static bool TryGetScalarValueOption(
+            string name,
+            [NotNullWhen(true)] out string? canonicalName)
+        {
+            if (CommandLineOptions.CanonicalNames.TryGetValue(name, out canonicalName))
+                return CommandLineOptions.ScalarValueOptionNames.Contains(canonicalName);
+
+            // Specified name is not a known option or it's not a scalar value option. 
+            canonicalName = null;
+            return false;
+        }
+
         private static bool Validate(CommandLineOptions options, ILogger logger)
         {
             if (options.BaseJob.IsBlank() || !AvailableJobs.ContainsKey(options.BaseJob))

tests/BenchmarkDotNet.Tests/CommandLineOptionsTests.cs

@@ -0,0 +1,76 @@
+using AwesomeAssertions;
+using BenchmarkDotNet.ConsoleArguments;
+using BenchmarkDotNet.Extensions;
+using CommandLine;
+using System.Reflection;
+
+namespace BenchmarkDotNet.Tests;
+
+public class CommandLineOptionsTests
+{
+    private sealed record OptionProperty(PropertyInfo Property, OptionAttribute Option);
+
+    /// <summary>
+    /// Validates that the hardcoded dictionaries in CommandLineOptions (CanonicalNames, MultiInstanceOptionNames)
+    /// are consistent with the actual option definitions in the CommandLineOptions class.
+    /// </summary>
+    [Fact]
+    public void ValidateCommandLineOptionDefinitions()
+    {
+        // Arrange
+        // Collect all option definitions of CommandLineOptions via reflection.
+        var optionProperties = typeof(CommandLineOptions)
+            .GetProperties(BindingFlags.Public | BindingFlags.Instance)
+            .Select(property => (Property: property, Option: property.GetCustomAttribute<OptionAttribute>()))
+            .Where(entry => entry.Option is not null)
+            .Select(x => new OptionProperty(x.Property, x.Option!))
+            .ToArray();
+
+        // Gets all long names.
+        var longNames = optionProperties
+            .Select(entry => entry.Option.LongName)
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+
+        // Gets all names (long and short)
+        var allNames = longNames.ToHashSet(StringComparer.OrdinalIgnoreCase);
+        foreach (var option in optionProperties.Where(x => x.Option.ShortName.IsNotBlank()))
+            allNames.Add(option.Option.ShortName);
+
+        // Act
+        var canonicalNames = CommandLineOptions.CanonicalNames;
+        var multiInstanceOptionNames = CommandLineOptions.MultiValueOptionNames;
+
+        // Assert
+        // Validate CanonicalNames definitions.
+        canonicalNames.Keys.Should().BeEquivalentTo(allNames);
+        canonicalNames.Values.Distinct().Should().BeEquivalentTo(longNames);
+
+        // Validate each short/long name resolves to its own canonical long name
+        // (Without this test, a swapped alias would pass the set checks above).
+        var expectedCanonicals = optionProperties
+            .SelectMany(entry => new[]
+            {
+                (Name: entry.Option!.LongName, Canonical: entry.Option.LongName),
+                (Name: entry.Option.ShortName, Canonical: entry.Option.LongName)
+            })
+            .Where(entry => entry.Name.IsNotBlank())
+            .ToArray();
+
+        foreach (var (name, expected) in expectedCanonicals)
+        {
+            bool isSuccess = canonicalNames.TryGetValue(name, out string? actual);
+            isSuccess.Should().BeTrue($"Name '{name}' should be resolve from CanonicalNames.");
+            actual.Should().Be(expected, $"Name '{name}' should resolve to '{expected}'.");
+        }
+
+        // Validate MultiInstanceOptionNames definitions are consistent with the property types.
+        foreach (var (property, option) in optionProperties)
+        {
+            bool isMultiValue = property.PropertyType != typeof(string) && typeof(System.Collections.IEnumerable).IsAssignableFrom(property.PropertyType);
+            if (isMultiValue)
+                multiInstanceOptionNames.Should().Contain(option.LongName);
+            else
+                multiInstanceOptionNames.Should().NotContain(option.LongName);
+        }
+    }
+}