[do not merge] Collect test assets on failed tests

azure-pipelines.yml

@@ -116,6 +116,13 @@ stages:
             displayName: Upload coverage to codecov.io
             condition: and(succeeded(), eq(variables._BuildConfig, 'Debug'))
 
+          - task: PublishBuildArtifacts@1
+            displayName: 'Publish test assets temp folders'
+            inputs:
+              PathtoPublish: '$(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/testsuite'
+              ArtifactName: TmpArtifacts_Windows_$(_BuildConfig)
+            condition: failed()
+
           # This step is only helpful for diagnosing some issues with vstest/test host that would not appear
           # through the console or trx
           - task: PublishBuildArtifacts@1
@@ -153,3 +160,10 @@ stages:
               ./test.sh --configuration $(_BuildConfig) --ci --test --integrationTest --nobl
             name: Test
             displayName: Tests
+
+          - task: PublishBuildArtifacts@1
+            displayName: 'Publish test assets temp folders'
+            inputs:
+              PathtoPublish: '$(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/testsuite'
+              ArtifactName: TmpArtifacts_Linux_$(_BuildConfig)
+            condition: failed()

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/NativeAotTests.cs

@@ -1,8 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-using System.Runtime.InteropServices;
-
 using Microsoft.Testing.Platform.Acceptance.IntegrationTests;
 using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers;
 
@@ -91,11 +89,6 @@ public NativeAotTests(ITestExecutionContext testExecutionContext, AcceptanceFixt
 
     public async Task NativeAotTests_WillRunWithExitCodeZero()
     {
-        if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-        {
-            return;
-        }
-
         using TestAsset generator = await TestAsset.GenerateAssetAsync(
            "NativeAotTests",
            SourceCode
@@ -106,23 +99,15 @@ public async Task NativeAotTests_WillRunWithExitCodeZero()
            .PatchCodeWithReplace("$MSTestEngineVersion$", MSTestEngineVersion),
            addPublicFeeds: true);
 
-        // The native AOT publication is pretty flaky and is often failing on CI with "fatal error LNK1136: invalid or corrupt file",
-        // or sometimes doesn't fail but the native code generation is not done.
-        // So, we retry the publication a few times.
-        await RetryHelper.RetryAsync(
-            async () =>
-            {
-                await DotnetCli.RunAsync(
-                    $"restore -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}",
-                    _acceptanceFixture.NuGetGlobalPackagesFolder.Path,
-                    retryCount: 0);
-                DotnetMuxerResult compilationResult = await DotnetCli.RunAsync(
-                    $"publish -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}",
-                    _acceptanceFixture.NuGetGlobalPackagesFolder.Path,
-                    retryCount: 0);
-                compilationResult.AssertOutputContains("Generating native code");
-            }, times: 15, every: TimeSpan.FromSeconds(5));
-
+        await DotnetCli.RunAsync(
+            $"restore -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}",
+            _acceptanceFixture.NuGetGlobalPackagesFolder.Path,
+            retryCount: 0);
+        DotnetMuxerResult compilationResult = await DotnetCli.RunAsync(
+            $"publish -m:1 -nodeReuse:false {generator.TargetAssetPath} -r {RID}",
+            _acceptanceFixture.NuGetGlobalPackagesFolder.Path,
+            retryCount: 0);
+        compilationResult.AssertOutputContains("Generating native code");
         var testHost = TestHost.LocateFrom(generator.TargetAssetPath, "NativeAotTests", TargetFrameworks.NetCurrent.Arguments, RID, Verb.publish);
 
         TestHostResult result = await testHost.ExecuteAsync();

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs

@@ -295,13 +295,8 @@ public async Task Invalid_TestingProfile_Name_Should_Fail(string multiTfm, Build
         compilationResult.AssertOutputContains("Invalid value for property TestingExtensionsProfile. Valid values are 'Default', 'AllMicrosoft' and 'None'.");
     }
 
-    public async Task NativeAot_Smoke_Test_Windows()
+    public async Task NativeAot_Smoke_Test()
     {
-        if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-        {
-            return;
-        }
-
         using TestAsset testAsset = await TestAsset.GenerateAssetAsync(
             AssetName,
             SingleTestSourceCode
@@ -313,20 +308,13 @@ public async Task NativeAot_Smoke_Test_Windows()
         """),
             addPublicFeeds: true);
 
-        // The native AOT publication is pretty flaky and is often failing on CI with "fatal error LNK1136: invalid or corrupt file",
-        // or sometimes doesn't fail but the native code generation is not done.
-        // So, we retry the publication a few times.
-        await RetryHelper.RetryAsync(
-            async () =>
-            {
-                DotnetMuxerResult compilationResult = await DotnetCli.RunAsync(
-                    $"publish -r {RID} -f {TargetFrameworks.NetCurrent.Arguments} {testAsset.TargetAssetPath}",
-                    _acceptanceFixture.NuGetGlobalPackagesFolder.Path,
-                    // We prefer to use the outer retry mechanism as we need some extra checks
-                    retryCount: 0);
-                compilationResult.AssertOutputContains("Generating native code");
-                compilationResult.AssertOutputNotContains("warning");
-            }, times: 15, every: TimeSpan.FromSeconds(5));
+        DotnetMuxerResult compilationResult = await DotnetCli.RunAsync(
+            $"publish -r {RID} -f {TargetFrameworks.NetCurrent.Arguments} {testAsset.TargetAssetPath}",
+            _acceptanceFixture.NuGetGlobalPackagesFolder.Path,
+            // We prefer to use the outer retry mechanism as we need some extra checks
+            retryCount: 0);
+        compilationResult.AssertOutputContains("Generating native code");
+        compilationResult.AssertOutputNotContains("warning");
 
         var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent.Arguments, verb: Verb.publish);
         TestHostResult testHostResult = await testHost.ExecuteAsync();