[main] Update dependencies to ensure coherency

NuGet.config

@@ -10,19 +10,15 @@
     <add key="darc-pub-dotnet-android-8db145f" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-android-8db145fa/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-android -->
     <!--  Begin: Package sources from dotnet-runtime -->
-    <add key="darc-pub-dotnet-runtime-df01702" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-runtime-df017025/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-runtime -->
     <!--  Begin: Package sources from dotnet-windowsdesktop -->
-    <add key="darc-pub-dotnet-windowsdesktop-7aba960" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-windowsdesktop-7aba9600/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-windowsdesktop -->
     <!--  Begin: Package sources from xamarin-xamarin-macios -->
     <add key="darc-pub-xamarin-xamarin-macios-b660cc6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-xamarin-xamarin-macios-b660cc67/nuget/v3/index.json" />
     <!--  End: Package sources from xamarin-xamarin-macios -->
     <!--  Begin: Package sources from dotnet-aspnetcore -->
-    <add key="darc-pub-dotnet-aspnetcore-bb179c2" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-aspnetcore-bb179c27/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-aspnetcore -->
     <!--  Begin: Package sources from dotnet-emsdk -->
-    <add key="darc-pub-dotnet-emsdk-8db661c" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-emsdk-8db661ce/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-emsdk -->
     <!--  Begin: Package sources from dotnet-maui -->
     <!--  End: Package sources from dotnet-maui -->

eng/Version.Details.xml

@@ -31,13 +31,13 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="10.0.0-beta.25268.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="10.0.0-beta.25374.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>35a34fa5ab9b2f97d3f7bdf48a7c2100727308b3</Sha>
+      <Sha>0fbc9e8711fa91e68c2bf3db376fb83b46f4ac35</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="10.0.0-beta.25268.1">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="10.0.0-beta.25374.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>35a34fa5ab9b2f97d3f7bdf48a7c2100727308b3</Sha>
+      <Sha>0fbc9e8711fa91e68c2bf3db376fb83b46f4ac35</Sha>
     </Dependency>
     <Dependency Name="Microsoft.DotNet.XHarness.CLI" Version="10.0.0-prerelease.25262.1">
       <Uri>https://github.com/dotnet/xharness</Uri>

eng/common/build.ps1

@@ -21,6 +21,7 @@ Param(
   [switch] $publish,
   [switch] $clean,
   [switch][Alias('pb')]$productBuild,
+  [switch]$fromVMR,
   [switch][Alias('bl')]$binaryLog,
   [switch][Alias('nobl')]$excludeCIBinarylog,
   [switch] $ci,
@@ -74,6 +75,7 @@ function Print-Usage() {
   Write-Host "  -nativeToolsOnMachine   Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)"
   Write-Host "  -nodeReuse <value>      Sets nodereuse msbuild parameter ('true' or 'false')"
   Write-Host "  -buildCheck             Sets /check msbuild parameter"
+  Write-Host "  -fromVMR                Set when building from within the VMR"
   Write-Host ""
 
   Write-Host "Command line arguments not listed above are passed thru to msbuild."
@@ -127,7 +129,8 @@ function Build {
     /p:Deploy=$deploy `
     /p:Test=$test `
     /p:Pack=$pack `
-    /p:DotNetBuildRepo=$productBuild `
+    /p:DotNetBuild=$productBuild `
+    /p:DotNetBuildFromVMR=$fromVMR `
     /p:IntegrationTest=$integrationTest `
     /p:PerformanceTest=$performanceTest `
     /p:Sign=$sign `

eng/common/build.sh

@@ -43,6 +43,7 @@ usage()
   echo "  --nodeReuse <value>      Sets nodereuse msbuild parameter ('true' or 'false')"
   echo "  --warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
   echo "  --buildCheck <value>     Sets /check msbuild parameter"
+  echo "  --fromVMR                Set when building from within the VMR"
   echo ""
   echo "Command line arguments not listed above are passed thru to msbuild."
   echo "Arguments can also be passed in with a single hyphen."
@@ -64,6 +65,7 @@ restore=false
 build=false
 source_build=false
 product_build=false
+from_vmr=false
 rebuild=false
 test=false
 integration_test=false
@@ -89,7 +91,7 @@ verbosity='minimal'
 runtime_source_feed=''
 runtime_source_feed_key=''
 
-properties=''
+properties=()
 while [[ $# > 0 ]]; do
   opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
@@ -142,6 +144,9 @@ while [[ $# > 0 ]]; do
       restore=true
       pack=true
       ;;
+    -fromvmr|-from-vmr)
+      from_vmr=true
+      ;;
     -test|-t)
       test=true
       ;;
@@ -187,7 +192,7 @@ while [[ $# > 0 ]]; do
       shift
       ;;
     *)
-      properties="$properties $1"
+      properties+=("$1")
       ;;
   esac
 
@@ -221,7 +226,7 @@ function Build {
   InitializeCustomToolset
 
   if [[ ! -z "$projects" ]]; then
-    properties="$properties /p:Projects=$projects"
+    properties+=("/p:Projects=$projects")
   fi
 
   local bl=""
@@ -241,8 +246,9 @@ function Build {
     /p:RepoRoot="$repo_root" \
     /p:Restore=$restore \
     /p:Build=$build \
-    /p:DotNetBuildRepo=$product_build \
+    /p:DotNetBuild=$product_build \
     /p:DotNetBuildSourceOnly=$source_build \
+    /p:DotNetBuildFromVMR=$from_vmr \
     /p:Rebuild=$rebuild \
     /p:Test=$test \
     /p:Pack=$pack \
@@ -251,7 +257,7 @@ function Build {
     /p:Sign=$sign \
     /p:Publish=$publish \
     /p:RestoreStaticGraphEnableBinaryLogger=$binary_log \
-    $properties
+    ${properties[@]+"${properties[@]}"}
 
   ExitWithExitCode 0
 }

eng/common/core-templates/job/job.yml

@@ -20,6 +20,7 @@ parameters:
   artifacts: ''
   enableMicrobuild: false
   enableMicrobuildForMacAndLinux: false
+  microbuildUseESRP: true
   enablePublishBuildArtifacts: false
   enablePublishBuildAssets: false
   enablePublishTestResults: false
@@ -128,6 +129,7 @@ jobs:
       parameters:
         enableMicrobuild: ${{ parameters.enableMicrobuild }}
         enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
+        microbuildUseESRP: ${{ parameters.microbuildUseESRP }}
         continueOnError: ${{ parameters.continueOnError }}
 
   - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:

eng/common/core-templates/job/onelocbuild.yml

@@ -86,8 +86,7 @@ jobs:
         isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
         ${{ if eq(parameters.CreatePr, true) }}:
           isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
-          ${{ if eq(parameters.RepoType, 'gitHub') }}:
-            isShouldReusePrSelected: ${{ parameters.ReusePr }}
+          isShouldReusePrSelected: ${{ parameters.ReusePr }}
         packageSourceAuth: patAuth
         patVariable: ${{ parameters.CeapexPat }}
         ${{ if eq(parameters.RepoType, 'gitHub') }}:
@@ -118,4 +117,4 @@ jobs:
           pathToPublish: '$(Build.SourcesDirectory)/eng/Localize/'
           publishLocation: Container
           artifactName: Loc
-          condition: ${{ parameters.condition }}
+          condition: ${{ parameters.condition }}

eng/common/core-templates/job/publish-build-assets.yml

@@ -32,6 +32,12 @@ parameters:
   # Optional: 🌤️ or not the build has assets it wants to publish to BAR
   isAssetlessBuild: false
 
+  # Optional, publishing version
+  publishingVersion: 3
+
+  # Optional: A minimatch pattern for the asset manifests to publish to BAR
+  assetManifestsPattern: '*/manifests/**/*.xml'
+
 jobs:
 - job: Asset_Registry_Publish
 
@@ -77,13 +83,31 @@ jobs:
       clean: true
 
     - ${{ if eq(parameters.isAssetlessBuild, 'false') }}: 
-      - task: DownloadPipelineArtifact@2
-        displayName: Download Asset Manifests
-        inputs:
-          artifactName: AssetManifests
-          targetPath: '$(Build.StagingDirectory)/AssetManifests'
-        condition: ${{ parameters.condition }}
-        continueOnError: ${{ parameters.continueOnError }}
+      - ${{ if eq(parameters.publishingVersion, 3) }}: 
+        - task: DownloadPipelineArtifact@2
+          displayName: Download Asset Manifests
+          inputs:
+            artifactName: AssetManifests
+            targetPath: '$(Build.StagingDirectory)/AssetManifests'
+          condition: ${{ parameters.condition }}
+          continueOnError: ${{ parameters.continueOnError }}
+      - ${{ if eq(parameters.publishingVersion, 4) }}:
+        - task: DownloadPipelineArtifact@2
+          displayName: Download V4 asset manifests
+          inputs:
+            itemPattern: '*/manifests/**/*.xml'
+            targetPath: '$(Build.StagingDirectory)/AllAssetManifests'
+          condition: ${{ parameters.condition }}
+          continueOnError: ${{ parameters.continueOnError }}
+        - task: CopyFiles@2
+          displayName: Copy V4 asset manifests to AssetManifests
+          inputs:
+            SourceFolder: '$(Build.StagingDirectory)/AllAssetManifests'
+            Contents: ${{ parameters.assetManifestsPattern }}
+            TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
+            flattenFolders: true
+          condition: ${{ parameters.condition }}
+          continueOnError: ${{ parameters.continueOnError }}
 
     - task: NuGetAuthenticate@1
 
@@ -120,6 +144,17 @@ jobs:
             Copy-Item -Path $symbolExclusionfile -Destination "$(Build.StagingDirectory)/ReleaseConfigs"
           }
 
+    - ${{ if eq(parameters.publishingVersion, 4) }}:
+      - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml
+        parameters:
+          is1ESPipeline: ${{ parameters.is1ESPipeline }}
+          args:
+            targetPath: '$(Build.ArtifactStagingDirectory)/MergedManifest.xml'
+            artifactName: AssetManifests
+            displayName: 'Publish Merged Manifest'
+            retryCountOnTaskFailure: 10 # for any logs being locked
+            sbomEnabled: false  # we don't need SBOM for logs
+
     - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
       parameters:
         is1ESPipeline: ${{ parameters.is1ESPipeline }}

eng/common/core-templates/jobs/jobs.yml

@@ -83,7 +83,6 @@ jobs:
   - template: /eng/common/core-templates/jobs/source-build.yml
     parameters:
       is1ESPipeline: ${{ parameters.is1ESPipeline }}
-      allCompletedJobId: Source_Build_Complete
       ${{ each parameter in parameters.sourceBuildParameters }}:
         ${{ parameter.key }}: ${{ parameter.value }}
 
@@ -108,8 +107,6 @@ jobs:
         - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
           - ${{ each job in parameters.jobs }}:
             - ${{ job.job }}
-        - ${{ if eq(parameters.enableSourceBuild, true) }}:
-          - Source_Build_Complete
 
         runAsPublic: ${{ parameters.runAsPublic }}
         publishAssetsImmediately: ${{ or(parameters.publishAssetsImmediately, parameters.isAssetlessBuild) }}

eng/common/core-templates/jobs/source-build.yml

@@ -2,12 +2,6 @@ parameters:
   # This template adds arcade-powered source-build to CI. A job is created for each platform, as
   # well as an optional server job that completes when all platform jobs complete.
 
-  # The name of the "join" job for all source-build platforms. If set to empty string, the job is
-  # not included. Existing repo pipelines can use this job depend on all source-build jobs
-  # completing without maintaining a separate list of every single job ID: just depend on this one
-  # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
-  allCompletedJobId: ''
-
   # See /eng/common/core-templates/job/source-build.yml
   jobNamePrefix: 'Source_Build'
 
@@ -31,16 +25,6 @@ parameters:
 
 jobs:
 
-- ${{ if ne(parameters.allCompletedJobId, '') }}:
-  - job: ${{ parameters.allCompletedJobId }}
-    displayName: Source-Build Complete
-    pool: server
-    dependsOn:
-    - ${{ each platform in parameters.platforms }}:
-      - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
-    - ${{ if eq(length(parameters.platforms), 0) }}:
-      - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
-
 - ${{ each platform in parameters.platforms }}:
   - template: /eng/common/core-templates/job/source-build.yml
     parameters:

eng/common/core-templates/post-build/post-build.yml

@@ -193,9 +193,6 @@ stages:
             buildId: $(AzDOBuildId)
             artifactName: PackageArtifacts
             checkDownloadedFiles: true
-            itemPattern: |
-              **
-              !**/Microsoft.SourceBuild.Intermediate.*.nupkg
 
         # This is necessary whenever we want to publish/restore to an AzDO private feed
         # Since sdk-task.ps1 tries to restore packages we need to do this authentication here

eng/common/core-templates/steps/install-microbuild.yml

@@ -4,8 +4,16 @@ parameters:
   # Enable install tasks for MicroBuild on Mac and Linux
   # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
   enableMicrobuildForMacAndLinux: false
+  # Determines whether the ESRP service connection information should be passed to the signing plugin.
+  # This overlaps with _SignType to some degree. We only need the service connection for real signing.
+  # It's important that the service connection not be passed to the MicroBuildSigningPlugin task in this place.
+  # Doing so will cause the service connection to be authorized for the pipeline, which isn't allowed and won't work for non-prod.
+  # Unfortunately, _SignType can't be used to exclude the use of the service connection in non-real sign scenarios. The
+  # variable is not available in template expression. _SignType has a very large proliferation across .NET, so replacing it is tough.
+  microbuildUseESRP: true
   # Location of the MicroBuild output folder
   microBuildOutputFolder: '$(Build.SourcesDirectory)'
+
   continueOnError: false
 
 steps:
@@ -21,15 +29,37 @@ steps:
           workingDirectory: ${{ parameters.microBuildOutputFolder }}
         condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
 
+    - script: |
+        REM Check if ESRP is disabled while SignType is real
+        if /I "${{ parameters.microbuildUseESRP }}"=="false" if /I "$(_SignType)"=="real" (
+          echo Error: ESRP must be enabled when SignType is real.
+          exit /b 1
+        )
+      displayName: 'Validate ESRP usage (Windows)'
+      condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
+    - script: |
+        # Check if ESRP is disabled while SignType is real
+        if [ "${{ parameters.microbuildUseESRP }}" = "false" ] && [ "$(_SignType)" = "real" ]; then
+          echo "Error: ESRP must be enabled when SignType is real."
+          exit 1
+        fi
+      displayName: 'Validate ESRP usage (Non-Windows)'
+      condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
+
     - task: MicroBuildSigningPlugin@4
       displayName: Install MicroBuild plugin
       inputs:
         signType: $(_SignType)
         zipSources: false
         feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
-        ${{ if and(eq(parameters.enableMicrobuildForMacAndLinux, 'true'), ne(variables['Agent.Os'], 'Windows_NT')) }}:
-          azureSubscription: 'MicroBuild Signing Task (DevDiv)'
-          useEsrpCli: true
+        ${{ if eq(parameters.microbuildUseESRP, true) }}:
+          ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}:
+            azureSubscription: 'MicroBuild Signing Task (DevDiv)'
+            useEsrpCli: true
+          ${{ elseif eq(variables['System.TeamProject'], 'DevDiv') }}:
+            ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea
+          ${{ else }}:
+            ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca
       env:
         TeamName: $(_TeamName)
         MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }}
@@ -48,38 +78,3 @@ steps:
             eq(variables['_SignType'], 'real')
           )
         ))
-
-    # Workaround for ESRP CLI on Linux - https://github.com/dotnet/source-build/issues/4964
-    - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}:
-      - task: UseDotNet@2
-        displayName: Install .NET 9.0 SDK for ESRP CLI Workaround
-        inputs:
-          packageType: sdk
-          version: 9.0.x
-          installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet
-          workingDirectory: ${{ parameters.microBuildOutputFolder }}
-        condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
-
-      - task: PowerShell@2
-        displayName: Workaround for ESRP CLI on Linux
-        inputs:
-          targetType: 'inline'
-          script: |
-            Write-Host "Copying Linux Path"
-            $MBSIGN_APPFOLDER = '$(MBSIGN_APPFOLDER)'
-            $MBSIGN_APPFOLDER = ($MBSIGN_APPFOLDER -replace '/build', '')
-
-            $versionRegex = '\d+\.\d+\.\d+'
-            $package = Get-ChildItem -Path $MBSIGN_APPFOLDER -Directory |
-              Where-Object { $_.Name -match $versionRegex }
-
-            if ($package.Count -ne 1) {
-                Write-Host "There should be exactly one matching subfolder, but found $($package.Count)."
-                exit 1
-            }
-
-            $MBSIGN_APPFOLDER = $package[0].FullName + '/build'
-            $MBSIGN_APPFOLDER | Write-Host
-            $SignConfigPath = $MBSIGN_APPFOLDER + '/signconfig.xml'
-            Copy-Item -Path "$(MBSIGN_APPFOLDER)/signconfig.xml" -Destination $SignConfigPath -Force
-        condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))