Skip redundant container uploads for existing manifests (#55690)

Author: jetersen

src/Containers/Microsoft.NET.Build.Containers/ImagePublisher.cs

@@ -12,6 +12,7 @@ public static async Task PublishImageAsync(
         BuiltImage singleArchImage,
         SourceImageReference sourceImageReference,
         DestinationImageReference destinationImageReference,
+        bool noCache,
         Microsoft.Build.Utilities.TaskLoggingHelper Log,
         Telemetry telemetry,
         CancellationToken cancellationToken)
@@ -37,7 +38,12 @@ await PushToRemoteRegistryAsync(
                     destinationImageReference,
                     Log,
                     cancellationToken,
-                    destinationImageReference.RemoteRegistry!.PushAsync,
+                    (image, source, destination, token) => destinationImageReference.RemoteRegistry!.PushAsync(
+                        image,
+                        source,
+                        destination,
+                        noCache,
+                        token),
                     Strings.ContainerBuilder_ImageUploadedToRegistry).ConfigureAwait(false);
                 break;
             default:

src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net11.0/PublicAPI.Unshipped.txt

@@ -257,6 +257,8 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.get -> b
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.SkipPublishing.get -> bool
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.SkipPublishing.set -> void
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.NoCache.get -> bool
+Microsoft.NET.Build.Containers.Tasks.CreateNewImage.NoCache.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedContainerNames.get -> Microsoft.Build.Framework.ITaskItem![]!
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedContainerNames.set -> void
 Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ImageFormat.get -> string?

src/Containers/Microsoft.NET.Build.Containers/Registry/DefaultManifestOperations.cs

@@ -24,6 +24,19 @@ internal DefaultManifestOperations(Uri baseUri, string registryName, HttpClient
         _registryName = registryName;
     }
 
+    public async Task<bool> ExistsAsync(string repositoryName, string reference, CancellationToken cancellationToken)
+    {
+        cancellationToken.ThrowIfCancellationRequested();
+        using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Head, new Uri(_baseUri, $"/v2/{repositoryName}/manifests/{reference}")).AcceptManifestFormats();
+        using HttpResponseMessage response = await _client.SendAsync(request, cancellationToken).ConfigureAwait(false);
+        return response.StatusCode switch
+        {
+            HttpStatusCode.OK => true,
+            _ when (int)response.StatusCode >= 500 => await LogAndThrowContainerHttpException<bool>(response, cancellationToken).ConfigureAwait(false),
+            _ => false,
+        };
+    }
+
     public async Task<HttpResponseMessage> GetAsync(string repositoryName, string reference, CancellationToken cancellationToken)
     {
         cancellationToken.ThrowIfCancellationRequested();

src/Containers/Microsoft.NET.Build.Containers/Registry/IManifestOperations.cs

@@ -11,6 +11,8 @@ namespace Microsoft.NET.Build.Containers;
 /// </remarks>
 internal interface IManifestOperations
 {
+    public Task<bool> ExistsAsync(string repositoryName, string reference, CancellationToken cancellationToken);
+
     public Task<HttpResponseMessage> GetAsync(string repositoryName, string reference, CancellationToken cancellationToken);
 
     public Task PutAsync(string repositoryName, string reference, string manifestListJson, string mediaType, CancellationToken cancellationToken);

src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs

@@ -595,13 +595,24 @@ public async Task PushManifestListAsync(
     }
 
     public Task PushAsync(BuiltImage builtImage, SourceImageReference source, DestinationImageReference destination, CancellationToken cancellationToken)
-        => PushAsync(builtImage, source, destination, pushTags: true, cancellationToken);
+        => PushAsync(builtImage, source, destination, noCache: false, cancellationToken);
 
-    private async Task PushAsync(BuiltImage builtImage, SourceImageReference source, DestinationImageReference destination, bool pushTags, CancellationToken cancellationToken)
+    public Task PushAsync(BuiltImage builtImage, SourceImageReference source, DestinationImageReference destination, bool noCache, CancellationToken cancellationToken)
+        => PushAsync(builtImage, source, destination, pushTags: true, noCache, cancellationToken);
+
+    private async Task PushAsync(BuiltImage builtImage, SourceImageReference source, DestinationImageReference destination, bool pushTags, bool noCache, CancellationToken cancellationToken)
     {
         cancellationToken.ThrowIfCancellationRequested();
         Registry destinationRegistry = destination.RemoteRegistry!;
 
+        bool manifestExists = !noCache &&
+            await _registryAPI.Manifest.ExistsAsync(destination.Repository, builtImage.ManifestDigest, cancellationToken).ConfigureAwait(false);
+
+        if (manifestExists)
+        {
+            _logger.LogInformation(Strings.Registry_ManifestExists, builtImage.ManifestDigest, destination.Repository);
+        }
+
         Func<Descriptor, Task> uploadLayerFunc = async (descriptor) =>
         {
             cancellationToken.ThrowIfCancellationRequested();
@@ -634,25 +645,28 @@ private async Task PushAsync(BuiltImage builtImage, SourceImageReference source,
             }
         };
 
-        if (SupportsParallelUploads)
+        if (!manifestExists)
         {
-            await Task.WhenAll(builtImage.LayerDescriptors.Select(descriptor => uploadLayerFunc(descriptor))).ConfigureAwait(false);
-        }
-        else
-        {
-            foreach (var descriptor in builtImage.LayerDescriptors)
+            if (SupportsParallelUploads)
             {
-                await uploadLayerFunc(descriptor).ConfigureAwait(false);
+                await Task.WhenAll(builtImage.LayerDescriptors.Select(descriptor => uploadLayerFunc(descriptor))).ConfigureAwait(false);
+            }
+            else
+            {
+                foreach (var descriptor in builtImage.LayerDescriptors)
+                {
+                    await uploadLayerFunc(descriptor).ConfigureAwait(false);
+                }
             }
-        }
 
-        cancellationToken.ThrowIfCancellationRequested();
-        using (MemoryStream stringStream = new(Encoding.UTF8.GetBytes(builtImage.Config)))
-        {
-            var configDigest = builtImage.ImageDigest!;
-            _logger.LogInformation(Strings.Registry_ConfigUploadStarted, configDigest);
-            await UploadBlobAsync(destination.Repository, configDigest, stringStream, cancellationToken).ConfigureAwait(false);
-            _logger.LogInformation(Strings.Registry_ConfigUploaded);
+            cancellationToken.ThrowIfCancellationRequested();
+            using (MemoryStream stringStream = new(Encoding.UTF8.GetBytes(builtImage.Config)))
+            {
+                var configDigest = builtImage.ImageDigest!;
+                _logger.LogInformation(Strings.Registry_ConfigUploadStarted, configDigest);
+                await UploadBlobAsync(destination.Repository, configDigest, stringStream, cancellationToken).ConfigureAwait(false);
+                _logger.LogInformation(Strings.Registry_ConfigUploaded);
+            }
         }
 
         // Tags can refer to an image manifest or an image manifest list.
@@ -668,7 +682,7 @@ private async Task PushAsync(BuiltImage builtImage, SourceImageReference source,
                 _logger.LogInformation(Strings.Registry_TagUploaded, tag, RegistryName);
             }
         }
-        else
+        else if (!manifestExists)
         {
             _logger.LogInformation(Strings.Registry_ManifestUploadStarted, RegistryName, builtImage.ManifestDigest);
             await _registryAPI.Manifest.PutAsync(destination.Repository, builtImage.ManifestDigest, builtImage.Manifest, builtImage.ManifestMediaType, cancellationToken).ConfigureAwait(false);

src/Containers/Microsoft.NET.Build.Containers/Resources/Strings.Designer.cs

@@ -453,6 +453,10 @@
     <value>Uploaded manifest to '{0}'.</value>
     <comment>{0} is the registry name</comment>
   </data>
+  <data name="Registry_ManifestExists" xml:space="preserve">
+    <value>Manifest '{0}' already exists in repository '{1}'. Skipping layer and configuration uploads.</value>
+    <comment>{0} is the manifest digest, {1} is the repository name</comment>
+  </data>
   <data name="Registry_ManifestUploadStarted" xml:space="preserve">
     <value>Uploading manifest to registry '{0}' as blob '{1}'.</value>
     <comment>{0} is the registry name</comment>