From 7aa95327e93944dfe6e20a99dece6795d80e68ef Mon Sep 17 00:00:00 2001 From: tmat Date: Fri, 5 Apr 2024 16:20:29 -0700 Subject: [PATCH 1/6] Set dotnet-watch timeout based on XUnitWorkItemTimeout --- ...omCreateXUnitWorkItemsWithTestExclusion.cs | 23 ++++++++++--------- .../Utilities/AwaitableProcess.cs | 7 ++++-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs index 87f95c449cb7..e16e372f863b 100644 --- a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs +++ b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs @@ -104,6 +104,14 @@ private async Task> PrepareWorkItem(ITaskItem xunitProject) xunitProject.TryGetMetadata("ExcludeAdditionalParameters", out string ExcludeAdditionalParameters); xunitProject.TryGetMetadata("Arguments", out string arguments); + TimeSpan timeout = TimeSpan.FromMinutes(5); + if (!string.IsNullOrEmpty(XUnitWorkItemTimeout)) + { + if (!TimeSpan.TryParse(XUnitWorkItemTimeout, out timeout)) + { + Log.LogWarning($"Invalid value \"{XUnitWorkItemTimeout}\" provided for XUnitWorkItemTimeout; falling back to default value of \"00:05:00\" (5 minutes)"); + } + } string assemblyName = Path.GetFileName(targetPath); @@ -136,30 +144,23 @@ private async Task> PrepareWorkItem(ITaskItem xunitProject) msbuildAdditionalSdkResolverFolder = ""; } + var commonArgs = $"{assemblyName} -e HELIX_WORK_ITEM_TIMEOUT={timeout} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} {(XUnitArguments != null ? " " + XUnitArguments : "")}"; + var scheduler = new AssemblyScheduler(methodLimit: !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TestFullMSBuild")) ? 32 : 16); var assemblyPartitionInfos = scheduler.Schedule(targetPath, netFramework: netFramework); var partitionedWorkItem = new List(); foreach (var assemblyPartitionInfo in assemblyPartitionInfos) { - string command = $"{driver} exec {assemblyName} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} {(XUnitArguments != null ? " " + XUnitArguments : "")} -xml testResults.xml {assemblyPartitionInfo.ClassListArgumentString} {arguments}"; + string command = $"{driver} exec {commonArgs} -xml testResults.xml {assemblyPartitionInfo.ClassListArgumentString} {arguments}"; if (netFramework) { var testFilter = string.IsNullOrEmpty(assemblyPartitionInfo.ClassListArgumentString) ? "" : $"--filter \"{assemblyPartitionInfo.ClassListArgumentString}\""; - command = $"{driver} test {assemblyName} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} {(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .\\ --logger trx {testFilter}"; + command = $"{driver} test {commonArgs} --results-directory .\\ --logger trx {testFilter}"; } Log.LogMessage($"Creating work item with properties Identity: {assemblyName}, PayloadDirectory: {publishDirectory}, Command: {command}"); - TimeSpan timeout = TimeSpan.FromMinutes(5); - if (!string.IsNullOrEmpty(XUnitWorkItemTimeout)) - { - if (!TimeSpan.TryParse(XUnitWorkItemTimeout, out timeout)) - { - Log.LogWarning($"Invalid value \"{XUnitWorkItemTimeout}\" provided for XUnitWorkItemTimeout; falling back to default value of \"00:05:00\" (5 minutes)"); - } - } - partitionedWorkItem.Add(new Microsoft.Build.Utilities.TaskItem(assemblyPartitionInfo.DisplayName + testIdentityDifferentiator, new Dictionary() { { "Identity", assemblyPartitionInfo.DisplayName + testIdentityDifferentiator}, diff --git a/test/dotnet-watch.Tests/Utilities/AwaitableProcess.cs b/test/dotnet-watch.Tests/Utilities/AwaitableProcess.cs index 200c21a0e5fa..77da26b85b4c 100644 --- a/test/dotnet-watch.Tests/Utilities/AwaitableProcess.cs +++ b/test/dotnet-watch.Tests/Utilities/AwaitableProcess.cs @@ -8,6 +8,10 @@ namespace Microsoft.DotNet.Watcher.Tools { internal class AwaitableProcess : IDisposable { + // cancel just before we hit timeout used on CI (XUnitWorkItemTimeout value in sdk\test\UnitTests.proj) + private static readonly TimeSpan s_timeout = Environment.GetEnvironmentVariable("HELIX_WORK_ITEM_TIMEOUT") is { } value + ? TimeSpan.Parse(value).Subtract(TimeSpan.FromSeconds(10)) : TimeSpan.FromMinutes(1); + private readonly object _testOutputLock = new(); private Process _process; @@ -70,8 +74,7 @@ public async Task GetOutputLineAsync(Predicate success, Predicat { using var cancellationOnFailure = new CancellationTokenSource(); - // cancel just before we hit 2 minute time out used on CI (sdk\test\UnitTests.proj) - cancellationOnFailure.CancelAfter(TimeSpan.FromSeconds(110)); + cancellationOnFailure.CancelAfter(s_timeout); var failedLineCount = 0; while (!_source.Completion.IsCompleted && failedLineCount == 0) From 6d18e4031c2807f2215a8a29ace337d6d99f97cb Mon Sep 17 00:00:00 2001 From: tmat Date: Fri, 5 Apr 2024 20:18:37 -0700 Subject: [PATCH 2/6] Fix --- ...SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs index e16e372f863b..91c8d4f938a0 100644 --- a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs +++ b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs @@ -144,19 +144,23 @@ private async Task> PrepareWorkItem(ITaskItem xunitProject) msbuildAdditionalSdkResolverFolder = ""; } - var commonArgs = $"{assemblyName} -e HELIX_WORK_ITEM_TIMEOUT={timeout} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} {(XUnitArguments != null ? " " + XUnitArguments : "")}"; - var scheduler = new AssemblyScheduler(methodLimit: !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TestFullMSBuild")) ? 32 : 16); var assemblyPartitionInfos = scheduler.Schedule(targetPath, netFramework: netFramework); var partitionedWorkItem = new List(); foreach (var assemblyPartitionInfo in assemblyPartitionInfos) { - string command = $"{driver} exec {commonArgs} -xml testResults.xml {assemblyPartitionInfo.ClassListArgumentString} {arguments}"; + string command; if (netFramework) { var testFilter = string.IsNullOrEmpty(assemblyPartitionInfo.ClassListArgumentString) ? "" : $"--filter \"{assemblyPartitionInfo.ClassListArgumentString}\""; - command = $"{driver} test {commonArgs} --results-directory .\\ --logger trx {testFilter}"; + command = $"{driver} test {assemblyName} -e HELIX_WORK_ITEM_TIMEOUT={timeout} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} " + + $"{(XUnitArguments != null ? " " + XUnitArguments : "")} --results-directory .\\ --logger trx {testFilter}"; + } + else + { + command = $"{driver} [env:HELIX_WORK_ITEM_TIMEOUT={timeout}] exec {assemblyName} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} " + + $"{(XUnitArguments != null ? " " + XUnitArguments : "")} -xml testResults.xml {assemblyPartitionInfo.ClassListArgumentString} {arguments}"; } Log.LogMessage($"Creating work item with properties Identity: {assemblyName}, PayloadDirectory: {publishDirectory}, Command: {command}"); From d7e64dcf8997409d8a7cfb5706909aa2654b684e Mon Sep 17 00:00:00 2001 From: tmat Date: Mon, 8 Apr 2024 10:02:29 -0700 Subject: [PATCH 3/6] Add support for setting env variables to test harness --- ...stomCreateXUnitWorkItemsWithTestExclusion.cs | 2 +- .../TestCommandLine.cs | 17 +++++++++++++++++ test/Microsoft.NET.TestFramework/TestContext.cs | 5 +++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs index 91c8d4f938a0..65f2b992a07d 100644 --- a/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs +++ b/test/HelixTasks/SDKCustomCreateXUnitWorkItemsWithTestExclusion.cs @@ -159,7 +159,7 @@ private async Task> PrepareWorkItem(ITaskItem xunitProject) } else { - command = $"{driver} [env:HELIX_WORK_ITEM_TIMEOUT={timeout}] exec {assemblyName} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} " + + command = $"{driver} exec {assemblyName} -e HELIX_WORK_ITEM_TIMEOUT={timeout} {testExecutionDirectory} {msbuildAdditionalSdkResolverFolder} " + $"{(XUnitArguments != null ? " " + XUnitArguments : "")} -xml testResults.xml {assemblyPartitionInfo.ClassListArgumentString} {arguments}"; } diff --git a/test/Microsoft.NET.TestFramework/TestCommandLine.cs b/test/Microsoft.NET.TestFramework/TestCommandLine.cs index f873ca8fab23..d0a86eb6e847 100644 --- a/test/Microsoft.NET.TestFramework/TestCommandLine.cs +++ b/test/Microsoft.NET.TestFramework/TestCommandLine.cs @@ -29,6 +29,8 @@ public class TestCommandLine public string MsbuildAdditionalSdkResolverFolder { get; set; } + public List<(string name, string value)> EnvironmentVariables { get; set; } + public List TestConfigFiles { get; private set; } = new List(); public HashSet TestListsToRun { get; private set; } = new HashSet(StringComparer.OrdinalIgnoreCase); @@ -91,6 +93,10 @@ public static TestCommandLine Parse(string[] args) { ret.TestListsToRun.Add(argStack.Pop()); } + else if (arg.Equals("-e", StringComparison.CurrentCultureIgnoreCase)) + { + ret.EnvironmentVariables.Add(ParseEnvironmentVariableArg(argStack.Pop())); + } else if (arg.Equals("-showSdkInfo", StringComparison.CurrentCultureIgnoreCase)) { ret.ShowSdkInfo = true; @@ -125,6 +131,17 @@ public static TestCommandLine Parse(string[] args) return ret; } + private static (string name, string value) ParseEnvironmentVariableArg(string arg) + { + var i = arg.IndexOf('='); + if (i <= 0) + { + throw new ArgumentException($"Invalid environment variable specification (expected 'name=value'): '{arg}'"); + } + + return (arg.Substring(0, i), arg.Substring(i + 1)); + } + public List GetXunitArgsFromTestConfig() { List testsToSkip = new(); diff --git a/test/Microsoft.NET.TestFramework/TestContext.cs b/test/Microsoft.NET.TestFramework/TestContext.cs index f7aae8e04cd4..93a5a47e1038 100644 --- a/test/Microsoft.NET.TestFramework/TestContext.cs +++ b/test/Microsoft.NET.TestFramework/TestContext.cs @@ -74,6 +74,11 @@ public static void Initialize(TestCommandLine commandLine) CommandLoggingContext.SetVerbose(true); Reporter.Reset(); + foreach (var (name, value) in commandLine.EnvironmentVariables) + { + Environment.SetEnvironmentVariable(name, value); + } + Environment.SetEnvironmentVariable("DOTNET_MULTILEVEL_LOOKUP", "0"); // Reset this environment variable so that if the dotnet under test is different than the From 9715584f13e2c1cb4bf7d23e9e5d4e2d353cf44c Mon Sep 17 00:00:00 2001 From: tmat Date: Mon, 8 Apr 2024 12:19:00 -0700 Subject: [PATCH 4/6] Fix --- test/Microsoft.NET.TestFramework/TestCommandLine.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.NET.TestFramework/TestCommandLine.cs b/test/Microsoft.NET.TestFramework/TestCommandLine.cs index d0a86eb6e847..2519a063e819 100644 --- a/test/Microsoft.NET.TestFramework/TestCommandLine.cs +++ b/test/Microsoft.NET.TestFramework/TestCommandLine.cs @@ -29,9 +29,9 @@ public class TestCommandLine public string MsbuildAdditionalSdkResolverFolder { get; set; } - public List<(string name, string value)> EnvironmentVariables { get; set; } + public List<(string name, string value)> EnvironmentVariables { get; set; } = []; - public List TestConfigFiles { get; private set; } = new List(); + public List TestConfigFiles { get; private set; } = []; public HashSet TestListsToRun { get; private set; } = new HashSet(StringComparer.OrdinalIgnoreCase); From 3089d1d4a21181af31cea09cd407d55e0dfdae53 Mon Sep 17 00:00:00 2001 From: tmat Date: Tue, 9 Apr 2024 11:30:46 -0700 Subject: [PATCH 5/6] Remove Program.UnitTests.cs --- ...ET.Build.Extensions.Tasks.UnitTests.csproj | 2 +- ...Microsoft.NET.Build.Tasks.UnitTests.csproj | 2 +- test/Common/Program.UnitTests.cs | 24 ------------------- 3 files changed, 2 insertions(+), 26 deletions(-) delete mode 100644 test/Common/Program.UnitTests.cs diff --git a/src/Tasks/Microsoft.NET.Build.Extensions.Tasks.UnitTests/Microsoft.NET.Build.Extensions.Tasks.UnitTests.csproj b/src/Tasks/Microsoft.NET.Build.Extensions.Tasks.UnitTests/Microsoft.NET.Build.Extensions.Tasks.UnitTests.csproj index 109d01102759..42ac6e1019e0 100644 --- a/src/Tasks/Microsoft.NET.Build.Extensions.Tasks.UnitTests/Microsoft.NET.Build.Extensions.Tasks.UnitTests.csproj +++ b/src/Tasks/Microsoft.NET.Build.Extensions.Tasks.UnitTests/Microsoft.NET.Build.Extensions.Tasks.UnitTests.csproj @@ -28,7 +28,7 @@ - + diff --git a/src/Tasks/Microsoft.NET.Build.Tasks.UnitTests/Microsoft.NET.Build.Tasks.UnitTests.csproj b/src/Tasks/Microsoft.NET.Build.Tasks.UnitTests/Microsoft.NET.Build.Tasks.UnitTests.csproj index 263fb5a44b49..7652a0865022 100644 --- a/src/Tasks/Microsoft.NET.Build.Tasks.UnitTests/Microsoft.NET.Build.Tasks.UnitTests.csproj +++ b/src/Tasks/Microsoft.NET.Build.Tasks.UnitTests/Microsoft.NET.Build.Tasks.UnitTests.csproj @@ -35,7 +35,7 @@ - + diff --git a/test/Common/Program.UnitTests.cs b/test/Common/Program.UnitTests.cs deleted file mode 100644 index 31f308383400..000000000000 --- a/test/Common/Program.UnitTests.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -partial class Program -{ - public static int Main(string[] args) - { - var newArgs = args.ToList(); - - // Help argument needs to be the first one to xunit, so don't insert assembly location in that case - if (args.Any(arg => arg.Equals("-help", StringComparison.CurrentCultureIgnoreCase) || arg.Equals("/?"))) - { - newArgs.Insert(0, "/?"); - } - else - { - newArgs.Insert(0, typeof(Program).Assembly.Location); - } - - int returnCode = Xunit.ConsoleClient.Program.Main(newArgs.ToArray()); - - return returnCode; - } -} From 9ff0b7d1f718a241e1e2bf6a96637e5ab12b78c9 Mon Sep 17 00:00:00 2001 From: tmat Date: Tue, 9 Apr 2024 13:19:01 -0700 Subject: [PATCH 6/6] Fixes --- .../Microsoft.NET.Build.Extensions.Tasks.UnitTests.csproj | 1 + test/Common/Program.cs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/Tasks/Microsoft.NET.Build.Extensions.Tasks.UnitTests/Microsoft.NET.Build.Extensions.Tasks.UnitTests.csproj b/src/Tasks/Microsoft.NET.Build.Extensions.Tasks.UnitTests/Microsoft.NET.Build.Extensions.Tasks.UnitTests.csproj index 42ac6e1019e0..3e0102c21879 100644 --- a/src/Tasks/Microsoft.NET.Build.Extensions.Tasks.UnitTests/Microsoft.NET.Build.Extensions.Tasks.UnitTests.csproj +++ b/src/Tasks/Microsoft.NET.Build.Extensions.Tasks.UnitTests/Microsoft.NET.Build.Extensions.Tasks.UnitTests.csproj @@ -24,6 +24,7 @@ + diff --git a/test/Common/Program.cs b/test/Common/Program.cs index 8b54b8bd40a0..d395586c7655 100644 --- a/test/Common/Program.cs +++ b/test/Common/Program.cs @@ -1,5 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.NET.TestFramework; +using Microsoft.NET.TestFramework.Commands; #pragma warning disable SA1205 // Partial elements should declare access partial class Program