diff --git a/eng/pipelines/pipelines/update-dependencies-internal.yml b/eng/pipelines/pipelines/update-dependencies-internal.yml index bc537e2d7a..31be1fa870 100644 --- a/eng/pipelines/pipelines/update-dependencies-internal.yml +++ b/eng/pipelines/pipelines/update-dependencies-internal.yml @@ -1,8 +1,9 @@ parameters: -# Stage container name (e.g., "stage-1234567") to fetch updates from. This can be -# from the real staging pipeline or the staging test pipeline, but the -# stagingStorageAccount parameter must match which pipeline is used here. -- name: stageContainer +# Comma-delimited list of stage container names (e.g., "stage-1234567,stage-2345678") +# to fetch updates from. This can be from the real staging pipeline or the staging +# test pipeline, but the stagingStorageAccount parameter must match which pipeline +# is used here. +- name: stageContainers type: string default: "" # Staging storage account for .NET release artifacts @@ -31,6 +32,8 @@ extends: parameters: dependencyName: dotnet updateSteps: + - ${{ if eq(parameters.stageContainers, '') }}: + - "stageContainers parameter must not be empty.": error - task: AzureCLI@2 displayName: Update .NET continueOnError: False @@ -42,7 +45,7 @@ extends: inlineScript: >- dotnet run --project eng/update-dependencies/update-dependencies.csproj -- from-staging-pipeline - ${{ parameters.stageContainer }} + "${{ parameters.stageContainers }}" --mode Remote --azdo-organization "$(System.CollectionUri)" --azdo-project "$(System.TeamProject)" diff --git a/eng/pipelines/update-dependencies-internal-official.yml b/eng/pipelines/update-dependencies-internal-official.yml index 56df087fe7..651e98625c 100644 --- a/eng/pipelines/update-dependencies-internal-official.yml +++ b/eng/pipelines/update-dependencies-internal-official.yml @@ -1,6 +1,14 @@ trigger: none pr: none +parameters: +# Comma-delimited list of stage container names (e.g., "stage-1234567,stage-2345678") +# Each stage container will be processed in sequence. +- name: stageContainers + type: string + default: "" + displayName: Comma-delimited list of stage containers to process + variables: - template: /eng/docker-tools/templates/variables/dotnet/common.yml@self # This pipeline expects there to be a variable named "targetBranch" defined @@ -9,19 +17,10 @@ variables: # It is not a parameter since it needs to be permanent but also changed to the # next release branch every month without modifying the pipeline YAML. -resources: - pipelines: - # All release pipelines are located at: https://dev.azure.com/dnceng/internal/_build?definitionScope=%5Cdotnet%5Crelease - # Stage-DotNet-Prepare-Artifacts: https://dev.azure.com/dnceng/internal/_build?definitionId=1300 - # Stage-DotNet-Prepare-Artifacts-Test: https://dev.azure.com/dnceng/internal/_build?definitionId=1286 - - pipeline: "dotnet-staging-pipeline" - source: "Stage-DotNet-Prepare-Artifacts" - trigger: true - extends: template: /eng/pipelines/pipelines/update-dependencies-internal.yml@self parameters: - stageContainer: "stage-$(resources.pipeline.dotnet-staging-pipeline.runID)" + stageContainers: "${{ parameters.stageContainers }}" stagingStorageAccount: "dotnetstage" targetBranch: "$(targetBranch)" gitServiceConnectionName: "$(updateDepsInt.serviceConnectionName)" diff --git a/eng/pipelines/update-dependencies-internal-unofficial.yml b/eng/pipelines/update-dependencies-internal-unofficial.yml index 44fafd52f2..6bacf7f6ae 100644 --- a/eng/pipelines/update-dependencies-internal-unofficial.yml +++ b/eng/pipelines/update-dependencies-internal-unofficial.yml @@ -6,25 +6,20 @@ parameters: type: string default: "nightly" displayName: Target branch for dependency update pull request. +# Comma-delimited list of stage container names (e.g., "stage-1234567,stage-2345678") +# Each stage container will be processed in sequence. +- name: stageContainers + type: string + default: "" + displayName: Comma-delimited list of stage containers to process variables: - template: /eng/docker-tools/templates/variables/dotnet/common.yml@self -resources: - pipelines: - # All release pipelines are located at: https://dev.azure.com/dnceng/internal/_build?definitionScope=%5Cdotnet%5Crelease - # Stage-DotNet-Prepare-Artifacts: https://dev.azure.com/dnceng/internal/_build?definitionId=1300 - # Stage-DotNet-Prepare-Artifacts-Test: https://dev.azure.com/dnceng/internal/_build?definitionId=1286 - - pipeline: "dotnet-staging-pipeline" - # Although this pipeline is "unofficial", it uses the production staging pipeline as a resource - # since we don't actually need to download any .NET artifacts from it in order to update this - # repo, so there's no risk in using the real versions. - source: "Stage-DotNet-Prepare-Artifacts" - extends: template: /eng/pipelines/pipelines/update-dependencies-internal.yml@self parameters: - stageContainer: "stage-$(resources.pipeline.dotnet-staging-pipeline.runID)" + stageContainers: "${{ parameters.stageContainers }}" stagingStorageAccount: "dotnetstage" targetBranch: "${{ parameters.targetBranch }}" gitServiceConnectionName: "$(updateDepsInt-test.serviceConnectionName)" diff --git a/eng/update-dependencies/FromStagingPipelineCommand.cs b/eng/update-dependencies/FromStagingPipelineCommand.cs index 7e5ede5e77..28c3e9e60e 100644 --- a/eng/update-dependencies/FromStagingPipelineCommand.cs +++ b/eng/update-dependencies/FromStagingPipelineCommand.cs @@ -11,10 +11,14 @@ namespace Dotnet.Docker; internal partial class FromStagingPipelineCommand : BaseCommand { /// - /// Callback that stages all changes, commits them, pushes them to the - /// remote, and creates a pull request. + /// Callback that stages all changes and commits them. /// - private delegate Task CommitAndCreatePullRequest(string commitMessage, string prTitle, string prBody); + private delegate Task CommitChanges(string commitMessage); + + /// + /// Callback that pushes all commits and creates a pull request. + /// + private delegate Task PushAndCreatePullRequest(string prTitle, string prBody); private readonly ILogger _logger; private readonly IPipelineArtifactProvider _pipelineArtifactProvider; @@ -44,17 +48,73 @@ public FromStagingPipelineCommand( public override async Task ExecuteAsync(FromStagingPipelineOptions options) { + var stageContainers = options.GetStageContainerList(); + + if (stageContainers.Count == 0) + { + _logger.LogError("No stage containers provided."); + return 1; + } + + _logger.LogInformation( + "Updating dependencies based on {Count} stage container(s): {StageContainers}", + stageContainers.Count, + string.Join(", ", stageContainers)); + // Delegate all git responsibilities to GitRepoContext. Depending on what options were // passed in, we may or may not want to actually perform git operations. GitRepoContext // decides what git operations to perform and tells us where to make changes. This keeps // all the git-related logic in one place. var gitRepoContext = await _createGitRepoContextAsync(options); - _logger.LogInformation( - "Updating dependencies based on stage container {StageContainer}", - options.StageContainer); + List commitMessages = []; + List prBodySections = []; + + // Process each stage container, creating a separate commit for each + foreach (var stageContainer in stageContainers) + { + _logger.LogInformation("Processing stage container: {StageContainer}", stageContainer); + + var (commitMessage, prBodySection, exitCode) = await ProcessStageContainerAsync( + options, + stageContainer, + gitRepoContext); + + if (exitCode != 0) + { + return exitCode; + } + + // Commit changes for this stage container + await gitRepoContext.CommitChanges(commitMessage); + + commitMessages.Add(commitMessage); + prBodySections.Add(prBodySection); + } - var stagingPipelineRunId = options.GetStagingPipelineRunId(); + // Create pull request with all commits + var prTitle = stageContainers.Count == 1 + ? $"[{options.TargetBranch}] {commitMessages[0]}" + : $"[{options.TargetBranch}] Update .NET dependencies from {stageContainers.Count} stage containers"; + + var prBody = string.Join(Environment.NewLine + Environment.NewLine, prBodySections); + await gitRepoContext.PushAndCreatePullRequest(prTitle, prBody); + + return 0; + } + + /// + /// Processes a single stage container and applies the updates. + /// + /// + /// A tuple containing the commit message, PR body section, and exit code. + /// + private async Task<(string CommitMessage, string PrBodySection, int ExitCode)> ProcessStageContainerAsync( + FromStagingPipelineOptions options, + string stageContainer, + GitRepoContext gitRepoContext) + { + var stagingPipelineRunId = StagingPipelineOptionsExtensions.GetStagingPipelineRunId(stageContainer); // Log staging pipeline tags for diagnostic purposes var stagingPipelineTags = await _pipelinesService.GetBuildTagsAsync( @@ -68,15 +128,15 @@ public override async Task ExecuteAsync(FromStagingPipelineOptions options) { ArgumentException.ThrowIfNullOrWhiteSpace( options.StagingStorageAccount, - $"{FromStagingPipelineOptions.StagingStorageAccountOption} must be set when using the {FromStagingPipelineOptions.InternalOption} option." + $"{FromStagingPipelineOptions.StagingStorageAccountOptionName} must be set when using the {FromStagingPipelineOptions.InternalOption} option." ); // Release metadata is stored in metadata/ReleaseManifest.json. // Release assets are stored individually under in assets/shipping/assets/[Sdk|Runtime|aspnetcore|...]. // Full example: https://dotnetstagetest.blob.core.windows.net/stage-2XXXXXX/assets/shipping/assets/Runtime/10.0.0-preview.N.XXXXX.YYY/dotnet-runtime-10.0.0-preview.N.XXXXX.YYY-linux-arm64.tar.gz - _buildLabelService.AddBuildTags($"Container - {options.StageContainer}"); + _buildLabelService.AddBuildTags($"Container - {stageContainer}"); internalBaseUrl = NormalizeStorageAccountUrl(options.StagingStorageAccount) - + $"/{options.StageContainer}/assets/shipping/assets"; + + $"/{stageContainer}/assets/shipping/assets"; } var releaseConfig = await _pipelineArtifactProvider.GetReleaseConfigAsync( @@ -94,7 +154,7 @@ public override async Task ExecuteAsync(FromStagingPipelineOptions options) _internalVersionsService.RecordInternalStagingBuild( repoRoot: gitRepoContext.LocalRepoPath, dotNetVersion: dotNetVersion, - stageContainer: options.StageContainer); + stageContainer: stageContainer); } var productVersions = (options.Internal, releaseConfig.SdkOnly) switch @@ -141,7 +201,7 @@ public override async Task ExecuteAsync(FromStagingPipelineOptions options) var buildUrl = $"{options.AzdoOrganization}/{options.AzdoProject}/_build/results?buildId={stagingPipelineRunId}"; _logger.LogInformation( "Applying internal build {StageContainer} ({BuildUrl})", - options.StageContainer, buildUrl); + stageContainer, buildUrl); _logger.LogInformation( "Ignore any git-related logging output below, because git " @@ -163,8 +223,8 @@ public override async Task ExecuteAsync(FromStagingPipelineOptions options) _logger.LogError( "Failed to apply stage container {StageContainer}. " + "Command exited with code {ExitCode}.", - options.StageContainer, exitCode); - return exitCode; + stageContainer, exitCode); + return (string.Empty, string.Empty, exitCode); } var commitMessage = releaseConfig switch @@ -173,18 +233,18 @@ public override async Task ExecuteAsync(FromStagingPipelineOptions options) _ => $"Update .NET {majorMinorVersionString} to {productVersions["sdk"]} SDK / {productVersions["runtime"]} Runtime", }; - var prTitle = $"[{options.TargetBranch}] {commitMessage}"; var newVersionsList = productVersions.Select(kvp => $"- {kvp.Key.ToUpper()}: {kvp.Value}"); - var prBody = $""" - This pull request updates .NET {majorMinorVersionString} to the following versions: + var prBodySection = $""" + ## .NET {majorMinorVersionString} + + This updates .NET {majorMinorVersionString} to the following versions: {string.Join(Environment.NewLine, newVersionsList)} - These versions are from .NET staging pipeline run [{options.StageContainer}]({buildUrl}). + These versions are from .NET staging pipeline run [{stageContainer}]({buildUrl}). """; - await gitRepoContext.CommitAndCreatePullRequest(commitMessage, prTitle, prBody); - return 0; + return (commitMessage, prBodySection, 0); } /// @@ -215,13 +275,18 @@ private static string NormalizeStorageAccountUrl(string storageAccount) /// Holds context about the git repository where changes should be made. /// /// Root of the repo where all changes should be made. - /// Callback that creates a pull request with all changes. - private record GitRepoContext(string LocalRepoPath, CommitAndCreatePullRequest CommitAndCreatePullRequest) + /// Callback that commits changes with the given message. + /// Callback that pushes all commits and creates a pull request. + private record GitRepoContext( + string LocalRepoPath, + CommitChanges CommitChanges, + PushAndCreatePullRequest PushAndCreatePullRequest) { /// /// Sets up the remote/local git repository based on . /// Call this before making any changes, then make changes to - /// and use to create a pull request. + /// and use to commit each change individually, + /// then use to push all commits and create a pull request. /// /// /// If is , @@ -233,7 +298,8 @@ public static async Task CreateAsync( FromStagingPipelineOptions options, IEnvironmentService environmentService) { - CommitAndCreatePullRequest createPullRequest; + CommitChanges commitChanges; + PushAndCreatePullRequest pushAndCreatePullRequest; string localRepoPath; if (options.Mode == ChangeMode.Remote) @@ -241,7 +307,12 @@ public static async Task CreateAsync( var remoteUrl = options.GetAzdoRepoUrl(); var targetBranch = options.TargetBranch; var buildId = environmentService.GetBuildId() ?? ""; - var prBranch = options.CreatePrBranchName($"update-deps-int-{options.StageContainer}", buildId); + var stageContainerList = options.GetStageContainerList(); + if (stageContainerList.Count == 0) + { + throw new ArgumentException("At least one stage container must be provided."); + } + var prBranch = options.CreatePrBranchName($"update-deps-int-{stageContainerList[0]}", buildId); var committer = options.GetCommitterIdentity(); // Clone the repo and configure git identity for commits @@ -253,10 +324,15 @@ public static async Task CreateAsync( await git.Local.CreateAndCheckoutLocalBranchAsync(prBranch); localRepoPath = git.Local.LocalPath; - createPullRequest = async (commitMessage, prTitle, prBody) => + + commitChanges = async (commitMessage) => { await git.Local.StageAsync("."); await git.Local.CommitAsync(commitMessage, committer); + }; + + pushAndCreatePullRequest = async (prTitle, prBody) => + { await git.PushLocalBranchAsync(prBranch); await git.Remote.CreatePullRequestAsync(new( Title: prTitle, @@ -270,16 +346,22 @@ await git.Remote.CreatePullRequestAsync(new( { logger.LogInformation("No git operations will be performed in {Mode} mode.", options.Mode); localRepoPath = options.RepoRoot; - createPullRequest = async (commitMessage, prTitle, prBody) => + + commitChanges = async (commitMessage) => { - logger.LogInformation("Skipping commit and pull request creation in {Mode} mode.", options.Mode); + logger.LogInformation("Skipping commit in {Mode} mode.", options.Mode); logger.LogInformation("Commit message: {CommitMessage}", commitMessage); + }; + + pushAndCreatePullRequest = async (prTitle, prBody) => + { + logger.LogInformation("Skipping push and pull request creation in {Mode} mode.", options.Mode); logger.LogInformation("Pull request title: {PullRequestTitle}", prTitle); logger.LogInformation("Pull request body:\n{PullRequestBody}", prBody); }; } - return new GitRepoContext(localRepoPath, createPullRequest); + return new GitRepoContext(localRepoPath, commitChanges, pushAndCreatePullRequest); } } } diff --git a/eng/update-dependencies/FromStagingPipelineOptions.cs b/eng/update-dependencies/FromStagingPipelineOptions.cs index d2bf427656..faeefe6987 100644 --- a/eng/update-dependencies/FromStagingPipelineOptions.cs +++ b/eng/update-dependencies/FromStagingPipelineOptions.cs @@ -12,9 +12,10 @@ internal partial record FromStagingPipelineOptions : CreatePullRequestOptions, I public const string InternalOption = "--internal"; /// - /// The stage container name (e.g., "stage-1234567") to use as a source for the update. + /// A comma-delimited list of stage container names (e.g., "stage-1234567,stage-2345678") + /// to use as a source for the update. /// - public required string StageContainer { get; init; } + public required string StageContainers { get; init; } /// /// Whether or not to use the internal versions of the staged build. @@ -34,10 +35,10 @@ internal partial record FromStagingPipelineOptions : CreatePullRequestOptions, I public static new List Arguments { get; } = [ - new Argument("stage-container") + new Argument("stage-containers") { Arity = ArgumentArity.ExactlyOne, - Description = "The stage container name to use as a source for the update (e.g., 'stage-1234567')" + Description = "A comma-delimited list of stage container names to use as a source for the update (e.g., 'stage-1234567,stage-2345678')" }, ..CreatePullRequestOptions.Arguments, ]; @@ -67,6 +68,16 @@ internal partial record FromStagingPipelineOptions : CreatePullRequestOptions, I }, ..CreatePullRequestOptions.Options, ]; + + /// + /// Parses the comma-delimited list of stage containers and returns them as a list. + /// + public IReadOnlyList GetStageContainerList() + { + return StageContainers + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToList(); + } } internal static partial class StagingPipelineOptionsExtensions @@ -80,13 +91,13 @@ internal static partial class StagingPipelineOptionsExtensions /// /// Thrown if the stage container name is not in the expected format. /// - public static int GetStagingPipelineRunId(this FromStagingPipelineOptions options) + public static int GetStagingPipelineRunId(string stageContainer) { - var match = StageContainerRegex.Match(options.StageContainer); + var match = StageContainerRegex.Match(stageContainer); if (!match.Success) { throw new ArgumentException( - $"Invalid stage container name '{options.StageContainer}'. Expected format: 'stage-{{buildId}}' (e.g., 'stage-1234567')"); + $"Invalid stage container name '{stageContainer}'. Expected format: 'stage-{{buildId}}' (e.g., 'stage-1234567')"); } return int.Parse(match.Groups[1].Value); } diff --git a/eng/update-dependencies/Sync/SyncInternalReleaseCommand.cs b/eng/update-dependencies/Sync/SyncInternalReleaseCommand.cs index b2d342eece..67b9b2a310 100644 --- a/eng/update-dependencies/Sync/SyncInternalReleaseCommand.cs +++ b/eng/update-dependencies/Sync/SyncInternalReleaseCommand.cs @@ -201,7 +201,7 @@ private async Task ApplyInternalBuildAsync( { RepoRoot = localRepo.LocalPath, Internal = true, - StageContainer = stageContainer, + StageContainers = stageContainer, StagingStorageAccount = stagingStorageAccount, AzdoOrganization = options.AzdoOrganization, AzdoProject = options.AzdoProject, diff --git a/tests/UpdateDependencies.Tests/FromStagingPipelineCommandTests.cs b/tests/UpdateDependencies.Tests/FromStagingPipelineCommandTests.cs index 345d5bd318..22f397275a 100644 --- a/tests/UpdateDependencies.Tests/FromStagingPipelineCommandTests.cs +++ b/tests/UpdateDependencies.Tests/FromStagingPipelineCommandTests.cs @@ -26,7 +26,7 @@ public async Task ExecuteAsync_InternalMode_AddsBuildTagWithStageContainer() var options = new FromStagingPipelineOptions { - StageContainer = stageContainer, + StageContainers = stageContainer, Internal = true, StagingStorageAccount = "https://dotnetstagetest.blob.core.windows.net/", Mode = ChangeMode.Local, @@ -50,7 +50,7 @@ public async Task ExecuteAsync_NotInternalMode_DoesNotAddBuildTag() var options = new FromStagingPipelineOptions { - StageContainer = "stage-1234567", + StageContainers = "stage-1234567", Internal = false, Mode = ChangeMode.Local, RepoRoot = "/tmp/repo" @@ -67,13 +67,7 @@ public async Task ExecuteAsync_NotInternalMode_DoesNotAddBuildTag() [InlineData("stage-999999999", 999999999)] public void GetStagingPipelineRunId_ValidStageContainer_ReturnsExpectedId(string stageContainer, int expectedId) { - var options = new FromStagingPipelineOptions - { - StageContainer = stageContainer, - RepoRoot = "/tmp/repo" - }; - - var result = options.GetStagingPipelineRunId(); + var result = StagingPipelineOptionsExtensions.GetStagingPipelineRunId(stageContainer); result.ShouldBe(expectedId); } @@ -85,16 +79,46 @@ public void GetStagingPipelineRunId_ValidStageContainer_ReturnsExpectedId(string [InlineData("stage-")] [InlineData("")] public void GetStagingPipelineRunId_InvalidStageContainer_ThrowsArgumentException(string stageContainer) + { + var exception = Should.Throw(() => + StagingPipelineOptionsExtensions.GetStagingPipelineRunId(stageContainer)); + exception.Message.ShouldContain($"Invalid stage container name '{stageContainer}'"); + exception.Message.ShouldContain("Expected format: 'stage-{buildId}'"); + } + + [Theory] + [InlineData("stage-1234567", new[] { "stage-1234567" })] + [InlineData("stage-1234567,stage-2345678", new[] { "stage-1234567", "stage-2345678" })] + [InlineData("stage-1234567, stage-2345678, stage-3456789", new[] { "stage-1234567", "stage-2345678", "stage-3456789" })] + [InlineData(" stage-1234567 , stage-2345678 ", new[] { "stage-1234567", "stage-2345678" })] + public void GetStageContainerList_ParsesCommaDelimitedList(string input, string[] expected) { var options = new FromStagingPipelineOptions { - StageContainer = stageContainer, + StageContainers = input, RepoRoot = "/tmp/repo" }; - var exception = Should.Throw(() => options.GetStagingPipelineRunId()); - exception.Message.ShouldContain($"Invalid stage container name '{stageContainer}'"); - exception.Message.ShouldContain("Expected format: 'stage-{buildId}'"); + var result = options.GetStageContainerList(); + + result.ShouldBe(expected); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData(",,,")] + public void GetStageContainerList_EmptyOrWhitespace_ReturnsEmptyList(string input) + { + var options = new FromStagingPipelineOptions + { + StageContainers = input, + RepoRoot = "/tmp/repo" + }; + + var result = options.GetStageContainerList(); + + result.ShouldBeEmpty(); } private static FromStagingPipelineCommand CreateCommand( diff --git a/tests/UpdateDependencies.Tests/SyncInternalReleaseTests.cs b/tests/UpdateDependencies.Tests/SyncInternalReleaseTests.cs index 352fb25012..d3e3487a79 100644 --- a/tests/UpdateDependencies.Tests/SyncInternalReleaseTests.cs +++ b/tests/UpdateDependencies.Tests/SyncInternalReleaseTests.cs @@ -249,7 +249,7 @@ public async Task Sync() fromStagingPipelineCommandMock.Verify(command => command.ExecuteAsync(It.Is(o => o.RepoRoot == LocalRepoPath - && o.StageContainer == stageContainer + && o.StageContainers == stageContainer && o.StagingStorageAccount == options.StagingStorageAccount )), Times.Once