[automated] Merge branch 'release/9.0.3xx' => 'main'

.github/workflows/remove-lockdown-label.yml

@@ -68,4 +68,4 @@ jobs:
                             name: 'Branch Lockdown'
                         });
                         console.log(`Removed Branch Lockdown label from PR #${pr.number}`);
-                    }
+                    }

NuGet.config

@@ -6,6 +6,7 @@
     <!--  Begin: Package sources from dotnet-aspire -->
     <!--  End: Package sources from dotnet-aspire -->
     <!--  Begin: Package sources from dotnet-runtime -->
+    <add key="darc-int-dotnet-runtime-3875b54" value="https://pkgs.dev.azure.com/dnceng/internal/_packaging/darc-int-dotnet-runtime-3875b54e/nuget/v3/index.json" />
     <!--  End: Package sources from dotnet-runtime -->
     <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
     <add key="dotnet6" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json" />
@@ -37,6 +38,7 @@
     <clear />
     <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
     <!--  Begin: Package sources from dotnet-runtime -->
+    <add key="darc-int-dotnet-runtime-3875b54" value="true" />
     <!--  End: Package sources from dotnet-runtime -->
     <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
   </disabledPackageSources>

src/Containers/Microsoft.NET.Build.Containers/ContainerBuilder.cs

@@ -216,6 +216,11 @@ private static async Task<int> PushToLocalRegistryAsync(ILogger logger, BuiltIma
             await containerRegistry.LoadAsync(builtImage, sourceImageReference, destinationImageReference, cancellationToken).ConfigureAwait(false);
             logger.LogInformation(Strings.ContainerBuilder_ImageUploadedToLocalDaemon, destinationImageReference, containerRegistry);
         }
+        catch (UnableToDownloadFromRepositoryException)
+        {
+            logger.LogError(Resource.FormatString(nameof(Strings.UnableToDownloadFromRepository)), sourceImageReference);
+            return 1;
+        }
         catch (Exception ex)
         {
             logger.LogError(Resource.FormatString(nameof(Strings.RegistryOutputPushFailed), ex.Message));
@@ -243,11 +248,6 @@ private static async Task<int> PushToRemoteRegistryAsync(ILogger logger, BuiltIm
             logger.LogError(Resource.FormatString(nameof(Strings.UnableToDownloadFromRepository)), sourceImageReference);
             return 1;
         }
-        catch (UnableToAccessRepositoryException)
-        {
-            logger.LogError(Resource.FormatString(nameof(Strings.UnableToAccessRepository), destinationImageReference.Repository, destinationImageReference.RemoteRegistry!.RegistryName));
-            return 1;
-        }
         catch (Exception e)
         {
             logger.LogError(Resource.FormatString(nameof(Strings.RegistryOutputPushFailed), e.Message));

src/Containers/Microsoft.NET.Build.Containers/ImagePublisher.cs

@@ -107,6 +107,10 @@ private static async Task PushToLocalRegistryAsync<T>(
             await loadFunc(image, sourceImageReference, destinationImageReference, cancellationToken).ConfigureAwait(false);
             Log.LogMessage(MessageImportance.High, Strings.ContainerBuilder_ImageUploadedToLocalDaemon, destinationImageReference, localRegistry);
         }
+        catch (UnableToDownloadFromRepositoryException)
+        {
+            Log.LogErrorWithCodeFromResources(nameof(Strings.UnableToDownloadFromRepository), sourceImageReference);
+        }
         catch (ContainerHttpException e)
         {
             Log.LogErrorFromException(e, true);

src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs

@@ -75,6 +75,8 @@ internal sealed class Registry
     private const string DockerHubRegistry1 = "registry-1.docker.io";
     private const string DockerHubRegistry2 = "registry.hub.docker.com";
     private static readonly int s_defaultChunkSizeBytes = 1024 * 64;
+    private const int MaxDownloadRetries = 5;
+    private readonly Func<TimeSpan> _retryDelayProvider;
 
     private readonly ILogger _logger;
     private readonly IRegistryAPI _registryAPI;
@@ -87,7 +89,7 @@ internal sealed class Registry
     /// </summary>
     public string RegistryName { get; }
 
-    internal Registry(string registryName, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null) :
+    internal Registry(string registryName, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null, Func<TimeSpan>? retryDelayProvider = null) :
         this(new Uri($"https://{registryName}"), logger, registryAPI, settings)
     { }
 
@@ -96,15 +98,15 @@ internal Registry(string registryName, ILogger logger, RegistryMode mode, Regist
     { }
 
 
-    internal Registry(Uri baseUri, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null) :
+    internal Registry(Uri baseUri, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null, Func<TimeSpan>? retryDelayProvider = null) :
         this(baseUri, logger, new RegistryApiFactory(registryAPI), settings)
     { }
 
     internal Registry(Uri baseUri, ILogger logger, RegistryMode mode, RegistrySettings? settings = null) :
         this(baseUri, logger, new RegistryApiFactory(mode), settings)
     { }
 
-    private Registry(Uri baseUri, ILogger logger, RegistryApiFactory factory, RegistrySettings? settings = null)
+    private Registry(Uri baseUri, ILogger logger, RegistryApiFactory factory, RegistrySettings? settings = null, Func<TimeSpan>? retryDelayProvider = null)
     {
         RegistryName = DeriveRegistryName(baseUri);
 
@@ -118,6 +120,8 @@ private Registry(Uri baseUri, ILogger logger, RegistryApiFactory factory, Regist
         _logger = logger;
         _settings = settings ?? new RegistrySettings(RegistryName);
         _registryAPI = factory.Create(RegistryName, BaseUri, logger, _settings.IsInsecure);
+
+        _retryDelayProvider = retryDelayProvider ?? (() => TimeSpan.FromSeconds(1));
     }
 
     private static string DeriveRegistryName(Uri baseUri)
@@ -404,33 +408,48 @@ public async Task<string> DownloadBlobAsync(string repository, Descriptor descri
     {
         cancellationToken.ThrowIfCancellationRequested();
         string localPath = ContentStore.PathForDescriptor(descriptor);
-
+    
         if (File.Exists(localPath))
         {
             // Assume file is up to date and just return it
             return localPath;
         }
-
+    
         string tempTarballPath = ContentStore.GetTempFile();
-
-        try
+
+        int retryCount = 0;
+        while (retryCount < MaxDownloadRetries)
         {
-            // No local copy, so download one
-            using Stream responseStream = await _registryAPI.Blob.GetStreamAsync(repository, descriptor.Digest, cancellationToken).ConfigureAwait(false);
-
-            using (FileStream fs = File.Create(tempTarballPath))
+            try
             {
-                await responseStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
+                // No local copy, so download one
+                using Stream responseStream = await _registryAPI.Blob.GetStreamAsync(repository, descriptor.Digest, cancellationToken).ConfigureAwait(false);
+
+                using (FileStream fs = File.Create(tempTarballPath))
+                {
+                    await responseStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
+                }
+
+                // Break the loop if successful
+                break;
+            }
+            catch (Exception ex)
+            {
+                retryCount++;
+                if (retryCount >= MaxDownloadRetries)
+                {
+                    throw new UnableToDownloadFromRepositoryException(repository);
+                }
+
+                _logger.LogTrace("Download attempt {0}/{1} for repository '{2}' failed. Error: {3}", retryCount, MaxDownloadRetries, repository, ex.ToString());
+
+                // Wait before retrying
+                await Task.Delay(_retryDelayProvider(), cancellationToken).ConfigureAwait(false);
             }
         }
-        catch (Exception)
-        {
-            throw new UnableToDownloadFromRepositoryException(repository);
-        }
-        cancellationToken.ThrowIfCancellationRequested();
-
+
         File.Move(tempTarballPath, localPath, overwrite: true);
-
+    
         return localPath;
     }
 

test/Microsoft.NET.Build.Containers.IntegrationTests/EndToEndTests.cs

@@ -4,6 +4,8 @@
 using System.Formats.Tar;
 using System.Runtime.CompilerServices;
 using System.Text.Json;
+using FakeItEasy;
+using Microsoft.Build.Framework;
 using Microsoft.DotNet.Cli.Utils;
 using Microsoft.NET.Build.Containers.LocalDaemons;
 using Microsoft.NET.Build.Containers.Resources;
@@ -1431,9 +1433,10 @@ static string[] DecideEntrypoint(string rid, string appName, string workingDir)
     }
 
     [DockerAvailableFact]
-    public async Task CheckErrorMessageWhenSourceRepositoryThrows()
+    public async void CheckDownloadErrorMessageWhenSourceRepositoryThrows()
     {
-        ILogger logger = _loggerFactory.CreateLogger(nameof(CheckErrorMessageWhenSourceRepositoryThrows));
+        var loggerFactory = new TestLoggerFactory(_testOutput);
+        var logger = loggerFactory.CreateLogger(nameof(CheckDownloadErrorMessageWhenSourceRepositoryThrows));
         string rid = "win-x64";
         string publishDirectory = BuildLocalApp(tfm: ToolsetInfo.CurrentTargetFramework, rid: rid);
 
@@ -1459,24 +1462,39 @@ public async Task CheckErrorMessageWhenSourceRepositoryThrows()
 
         // Load the image into the local registry
         var sourceReference = new SourceImageReference(registry, "some_random_image", DockerRegistryManager.Net9ImageTag, null);
-        var destinationReference = new DestinationImageReference(registry, NewImageName(), new[] { rid });
-        var sawMyException = false;
-        try
-        {
-            await new DockerCli(_loggerFactory).LoadAsync(builtImage, sourceReference, destinationReference, default).ConfigureAwait(false);
-        }
-        catch (UnableToDownloadFromRepositoryException e)
-        {
-            sawMyException = true;
-            Assert.Contains("The download of the image from repository some_random_image has failed", e.ToString());
-        }
-        Assert.True(sawMyException);
+        string archivePath = Path.Combine(TestSettings.TestArtifactsDirectory, nameof(CheckDownloadErrorMessageWhenSourceRepositoryThrows));
+        var destinationReference = new DestinationImageReference(new ArchiveFileRegistry(archivePath), NewImageName(), new[] { rid });
+
+        (var taskLog, var errors) = SetupTaskLog();
+        var telemetry = new Telemetry(sourceReference, destinationReference, taskLog);
+
+        await ImagePublisher.PublishImageAsync(builtImage, sourceReference, destinationReference, taskLog, telemetry, CancellationToken.None)
+                .ConfigureAwait(false);
+
+        // Assert the error message
+        Assert.True(taskLog.HasLoggedErrors);
+        Assert.NotNull(errors);
+        Assert.Single(errors);
+        Assert.Contains("Unable to download image from the repository", errors[0]);
 
         static string[] DecideEntrypoint(string rid, string appName, string workingDir)
         {
             var binary = rid.StartsWith("win", StringComparison.Ordinal) ? $"{appName}.exe" : appName;
             return new[] { $"{workingDir}/{binary}" };
         }
+
+        static (Microsoft.Build.Utilities.TaskLoggingHelper, List<string?> errors) SetupTaskLog()
+        {
+            // We can use any Task, we just need TaskLoggingHelper
+            Tasks.CreateNewImage cni = new();
+            List<string?> errors = new();
+            IBuildEngine buildEngine = A.Fake<IBuildEngine>();
+            A.CallTo(() => buildEngine.LogWarningEvent(A<BuildWarningEventArgs>.Ignored)).Invokes((BuildWarningEventArgs e) => errors.Add(e.Message));
+            A.CallTo(() => buildEngine.LogErrorEvent(A<BuildErrorEventArgs>.Ignored)).Invokes((BuildErrorEventArgs e) => errors.Add(e.Message));
+            A.CallTo(() => buildEngine.LogMessageEvent(A<BuildMessageEventArgs>.Ignored)).Invokes((BuildMessageEventArgs e) => errors.Add(e.Message));
+            cni.BuildEngine = buildEngine;
+            return (cni.Log, errors);
+        }
     }
 
     [DockerAvailableFact(checkContainerdStoreAvailability: true)]

test/Microsoft.NET.Build.Containers.UnitTests/RegistryTests.cs

@@ -545,6 +545,77 @@ public void IsRegistryInsecure(string registryName, string? insecureRegistriesEn
         Assert.Equal(expectedInsecure, registrySettings.IsInsecure);
     }
 
+    [Fact]
+    public async Task DownloadBlobAsync_RetriesOnFailure()
+    {
+        // Arrange
+        var logger = _loggerFactory.CreateLogger(nameof(DownloadBlobAsync_RetriesOnFailure));
+
+        var repoName = "testRepo";
+        var descriptor = new Descriptor(SchemaTypes.OciLayerGzipV1, "sha256:testdigest1234", 1234);
+        var cancellationToken = CancellationToken.None;
+
+        var mockRegistryAPI = new Mock<IRegistryAPI>(MockBehavior.Strict);
+        mockRegistryAPI
+            .SetupSequence(api => api.Blob.GetStreamAsync(repoName, descriptor.Digest, cancellationToken))
+            .ThrowsAsync(new Exception("Simulated failure 1")) // First attempt fails
+            .ThrowsAsync(new Exception("Simulated failure 2")) // Second attempt fails
+            .ReturnsAsync(new MemoryStream(new byte[] { 1, 2, 3 })); // Third attempt succeeds
+
+        Registry registry = new(repoName, logger, mockRegistryAPI.Object, null, () => TimeSpan.Zero);
+
+        string? result = null;
+        try
+        {
+            // Act
+            result = await registry.DownloadBlobAsync(repoName, descriptor, cancellationToken);
+
+            // Assert
+            Assert.NotNull(result);
+            Assert.True(File.Exists(result)); // Ensure the file was successfully downloaded
+            mockRegistryAPI.Verify(api => api.Blob.GetStreamAsync(repoName, descriptor.Digest, cancellationToken), Times.Exactly(3)); // Verify retries
+        }
+        finally
+        {
+            // Cleanup
+            if (result != null)
+            {
+                File.Delete(result);
+            }
+        }
+    }
+
+    [Fact]
+    public async Task DownloadBlobAsync_ThrowsAfterMaxRetries()
+    {
+        // Arrange
+        var logger = _loggerFactory.CreateLogger(nameof(DownloadBlobAsync_ThrowsAfterMaxRetries));
+
+        var repoName = "testRepo";
+        var descriptor = new Descriptor(SchemaTypes.OciLayerGzipV1, "sha256:testdigest1234", 1234);
+        var cancellationToken = CancellationToken.None;
+
+        var mockRegistryAPI = new Mock<IRegistryAPI>(MockBehavior.Strict);
+        // Simulate 5 failures (assuming your retry logic attempts 5 times before throwing)
+        mockRegistryAPI
+            .SetupSequence(api => api.Blob.GetStreamAsync(repoName, descriptor.Digest, cancellationToken))
+            .ThrowsAsync(new Exception("Simulated failure 1"))
+            .ThrowsAsync(new Exception("Simulated failure 2"))
+            .ThrowsAsync(new Exception("Simulated failure 3"))
+            .ThrowsAsync(new Exception("Simulated failure 4"))
+            .ThrowsAsync(new Exception("Simulated failure 5"));
+
+        Registry registry = new(repoName, logger, mockRegistryAPI.Object, null, () => TimeSpan.Zero);
+
+        // Act & Assert
+        await Assert.ThrowsAsync<UnableToDownloadFromRepositoryException>(async () =>
+        {
+            await registry.DownloadBlobAsync(repoName, descriptor, cancellationToken);
+        });
+
+        mockRegistryAPI.Verify(api => api.Blob.GetStreamAsync(repoName, descriptor.Digest, cancellationToken), Times.Exactly(5));
+    }
+
     private static NextChunkUploadInformation ChunkUploadSuccessful(Uri requestUri, Uri uploadUrl, int? contentLength, HttpStatusCode code = HttpStatusCode.Accepted)
     {
         return new(uploadUrl);