Automate SPMI collection for TE benchmarks (#119098)

Author: EgorBo

eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml

@@ -101,6 +101,9 @@ jobs:
     - ${{ if eq(parameters.collectionName, 'realworld') }}:
       - name: InputDirectory
         value: '$(Core_Root_Dir)'
+    - ${{ if eq(parameters.collectionName, 'aspnet2') }}:
+      - name: InputDirectory
+        value: '$(Core_Root_Dir)'
     - ${{ if eq(parameters.collectionName, 'coreclr_tests') }}:
       - name: InputDirectory
         value: '$(managedTestArtifactRootFolderPath)'

eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml

@@ -480,3 +480,24 @@ extends:
             unifiedBuildConfigOverride: checked
             # Default timeout is 150 minutes, which is too low for osx-arm64 queue.
             timeoutInMinutes: 300
+
+      #
+      # Collection of aspnet/TechEmpower benchmarks (crank-based): aspnet2
+      #
+      - template: /eng/pipelines/common/platform-matrix.yml
+        parameters:
+          jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml
+          buildConfig: checked
+          platforms:
+          - osx_arm64
+          - linux_arm64
+          # - linux_x64 # currently runs of disk space too often.
+          - windows_x64
+          - windows_arm64
+          helixQueueGroup: ci
+          helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+          jobParameters:
+            testGroup: outerloop
+            liveLibrariesBuildConfig: Release
+            collectionType: run
+            collectionName: aspnet2

src/coreclr/scripts/superpmi-collect.proj

@@ -68,10 +68,10 @@
 
   <!-- OS-specific settings for non-benchmarks collections -->
 
-  <PropertyGroup Condition=" '$(CollectionName)' != 'benchmarks' and '$(CollectionName)' != 'realworld' and '$(AGENT_OS)' == 'Windows_NT' ">
+  <PropertyGroup Condition=" '$(CollectionName)' != 'benchmarks' and '$(CollectionName)' != 'realworld' and '$(CollectionName)' != 'aspnet2' and '$(AGENT_OS)' == 'Windows_NT' ">
     <AssembliesDirectoryOnHelix>%HELIX_WORKITEM_PAYLOAD%\binaries</AssembliesDirectoryOnHelix>
   </PropertyGroup>
-  <PropertyGroup Condition=" '$(CollectionName)' != 'benchmarks' and '$(CollectionName)' != 'realworld' and '$(AGENT_OS)' != 'Windows_NT' ">
+  <PropertyGroup Condition=" '$(CollectionName)' != 'benchmarks' and '$(CollectionName)' != 'realworld' and '$(CollectionName)' != 'aspnet2' and '$(AGENT_OS)' != 'Windows_NT' ">
     <AssembliesDirectoryOnHelix>$HELIX_WORKITEM_PAYLOAD/binaries</AssembliesDirectoryOnHelix>
   </PropertyGroup>
 
@@ -84,6 +84,15 @@
     <PerformanceDirectory>$HELIX_WORKITEM_PAYLOAD/performance</PerformanceDirectory>
   </PropertyGroup>
 
+  <!-- OS-specific settings for aspnet2 collections -->
+
+  <PropertyGroup Condition=" '$(CollectionName)' == 'aspnet2' and '$(AGENT_OS)' == 'Windows_NT' ">
+    <CrankWorkDirectory>%HELIX_WORKITEM_PAYLOAD%</CrankWorkDirectory>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(CollectionName)' == 'aspnet2' and '$(AGENT_OS)' != 'Windows_NT' ">
+    <CrankWorkDirectory>$HELIX_WORKITEM_PAYLOAD</CrankWorkDirectory>
+  </PropertyGroup>
+
   <!-- Common non-OS-specific settings -->
 
   <PropertyGroup>
@@ -138,7 +147,7 @@
 
   <!-- Define the Helix work item command and timeout for each collection type -->
 
-  <PropertyGroup Condition="'$(CollectionName)' != 'benchmarks' and '$(CollectionName)' != 'realworld'">
+  <PropertyGroup Condition="'$(CollectionName)' != 'benchmarks' and '$(CollectionName)' != 'realworld' and '$(CollectionName)' != 'aspnet2'">
     <WorkItemCommand>$(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi.py collect --clean -log_level DEBUG --$(CollectionType) $(PmiArguments) $(InputKind) $(AssembliesDirectoryOnHelix) -arch $(Architecture) -build_type $(BuildConfig) -core_root $(SuperPMIDirectory)</WorkItemCommand>
     <WorkItemTimeout>2:00</WorkItemTimeout>
   </PropertyGroup>
@@ -148,6 +157,11 @@
     <WorkItemTimeout>3:00</WorkItemTimeout>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(CollectionName)' == 'aspnet2'">
+    <WorkItemCommand>$(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet2.py --core_root $(SuperPMIDirectory) --work_dir $(CrankWorkDirectory) </WorkItemCommand>
+    <WorkItemTimeout>3:00</WorkItemTimeout>
+  </PropertyGroup>
+
   <PropertyGroup>
     <EnableAzurePipelinesReporter>false</EnableAzurePipelinesReporter>
     <EnableXUnitReporter>false</EnableXUnitReporter>
@@ -157,6 +171,18 @@
   <ItemGroup Condition=" '$(AGENT_OS)' == 'Windows_NT' ">
     <HelixPreCommand Include="taskkill.exe /f /im corerun.exe"/>
     <HelixPostCommand Include="taskkill.exe /f /im corerun.exe"/>
+
+    <HelixPreCommand Include="taskkill.exe /f /im crank-agent.exe"/>
+    <HelixPostCommand Include="taskkill.exe /f /im crank-agent.exe"/>
+  </ItemGroup>
+  
+  <ItemGroup Condition=" '$(AGENT_OS)' != 'Windows_NT'">
+    <HelixPreCommand Include="pkill -f crank || true"/>
+    <HelixPostCommand Include="pkill -f crank || true"/>
+
+    <!-- Temp workaround, will be removed once https://github.com/dotnet/crank/pull/841 lands -->
+    <HelixPreCommand Include="sudo tdnf install git -y"/>
+    <HelixPreCommand Include="sudo tdnf update -y"/>
   </ItemGroup>
 
   <!-- We're currently not using .dotnet on the Helix agents -->
@@ -197,7 +223,7 @@
   <!-- For non-benchmark collections, we create one Helix work item for each sub-directory
        in the `InputArtifacts` directory.
   -->
-  <ItemGroup Condition="'$(CollectionName)' != 'benchmarks' and '$(CollectionName)' != 'realworld'">
+  <ItemGroup Condition="'$(CollectionName)' != 'benchmarks' and '$(CollectionName)' != 'realworld' and '$(CollectionName)' != 'aspnet2'">
     <PartitionDirectories Include="$([System.IO.Directory]::GetDirectories($(InputArtifacts)))"/>
     <Partition Include="@(PartitionDirectories -> '%(Filename)')" CollectAssemblies="%(Filename)" PartitionId="%(Filename)" />
   </ItemGroup>
@@ -262,9 +288,13 @@
     <BDN_Partition Include="Partition3" Index="3" BenchmarkPath="src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj" BenchmarkBinary="Microsoft.ML.Benchmarks.dll" />
     <BDN_Partition Include="Partition4" Index="4" BenchmarkPath="src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj" BenchmarkBinary="CompilerBenchmarks.dll" />
     <BDN_Partition Include="Partition5" Index="5" BenchmarkPath="src/benchmarks/real-world/PowerShell.Benchmarks/PowerShell.Benchmarks.csproj" BenchmarkBinary="PowerShell.Benchmarks.dll" />
-  </ItemGroup>  
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(CollectionName)' == 'aspnet2'">
+    <Crank_Partition Include="Partition0" Index="0" />
+  </ItemGroup>
 
-  <ItemGroup Condition="'$(CollectionName)' != 'benchmarks' and '$(CollectionName)' != 'realworld'">
+  <ItemGroup Condition="'$(CollectionName)' != 'benchmarks' and '$(CollectionName)' != 'realworld' and '$(CollectionName)' != 'aspnet2'">
     <HelixWorkItem Include="@(Partition)">
       <OutputFileName>$(CollectionName).$(CollectionType).%(HelixWorkItem.PartitionId).$(MchFileTag)</OutputFileName>
       <PayloadDirectory>$(AssembliesPayload)$(FileSeparatorChar)%(HelixWorkItem.CollectAssemblies)</PayloadDirectory>
@@ -292,5 +322,15 @@
       <Timeout>$(WorkItemTimeout)</Timeout>
       <DownloadFilesFromResults>%(OutputFileName).mch;%(OutputFileName).mch.mct;%(OutputFileName).log</DownloadFilesFromResults>
     </HelixWorkItem>
-  </ItemGroup>  
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(CollectionName)' == 'aspnet2'">
+    <HelixWorkItem Include="@(Crank_Partition)">
+      <OutputFileName>$(CollectionName).$(CollectionType).%(HelixWorkItem.Index).$(MchFileTag)</OutputFileName>
+      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <Command>$(WorkItemCommand) --output_mch $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch</Command>
+      <Timeout>$(WorkItemTimeout)</Timeout>
+      <DownloadFilesFromResults>%(OutputFileName).mch;%(OutputFileName).mch.mct</DownloadFilesFromResults>
+    </HelixWorkItem>
+  </ItemGroup> 
 </Project>