Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallelize workload pack downloads Fixes #43870 #45969

Merged
merged 5 commits into from
Feb 14, 2025
Merged

Parallelize workload pack downloads Fixes #43870 #45969

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
adf5458
Parallelize workload pack downloads
Forgind Jan 14, 2025
01175f7
Show stack trace and fix concurrency issue?
Forgind Jan 22, 2025
911866d
Revert stack changes
Forgind Jan 23, 2025
0ad86bd
One more
Forgind Jan 23, 2025
55dd0f2
Little fix
Forgind Feb 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/Cli/dotnet/NugetPackageDownloader/NuGetPackageDownloader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Collections.Concurrent;
using Microsoft.DotNet.Cli.Utils;
using Microsoft.DotNet.ToolPackage;
using Microsoft.DotNet.Tools;
Expand Down Expand Up @@ -35,7 +36,7 @@ internal class NuGetPackageDownloader : INuGetPackageDownloader
private readonly IReporter _reporter;
private readonly IFirstPartyNuGetPackageSigningVerifier _firstPartyNuGetPackageSigningVerifier;
private bool _validationMessagesDisplayed = false;
private readonly Dictionary<PackageSource, SourceRepository> _sourceRepositories;
private readonly ConcurrentDictionary<PackageSource, SourceRepository> _sourceRepositories;
private readonly bool _shouldUsePackageSourceMapping;

private readonly bool _verifySignatures;
Expand Down Expand Up @@ -818,7 +819,7 @@ private SourceRepository GetSourceRepository(PackageSource source)
if (!_sourceRepositories.TryGetValue(source, out SourceRepository value))
{
value = Repository.Factory.GetCoreV3(source);
_sourceRepositories.Add(source, value);
_sourceRepositories.AddOrUpdate(source, _ => value, (_, _) => value);
}

return value;
Expand Down
118 changes: 62 additions & 56 deletions src/Cli/dotnet/commands/dotnet-workload/install/FileBasedInstaller.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Collections.Concurrent;
using System.Text.Json;
using Microsoft.DotNet.Cli;
using Microsoft.DotNet.Cli.NuGetPackageDownloader;
Expand Down Expand Up @@ -140,89 +141,94 @@ public void InstallWorkloads(IEnumerable<WorkloadId> workloadIds, SdkFeatureBand
public void InstallWorkloads(IEnumerable<WorkloadId> workloadIds, SdkFeatureBand sdkFeatureBand, ITransactionContext transactionContext, bool overwriteExistingPacks, DirectoryPath? offlineCache = null)
{
var packInfos = GetPacksInWorkloads(workloadIds);

List<PackInfo> packsToInstall = [];
foreach (var packInfo in packInfos)
{
if (PackIsInstalled(packInfo) && !overwriteExistingPacks)
{
_reporter.WriteLine(string.Format(LocalizableStrings.WorkloadPackAlreadyInstalledMessage, packInfo.ResolvedPackageId, packInfo.Version));
WritePackInstallationRecord(packInfo, sdkFeatureBand);
}
else
{
packsToInstall.Add(packInfo);
}
}

var tempFilesToDelete = new ConcurrentBag<string>();
var packagePaths = packsToInstall.AsParallel().Select(packInfo =>
{
if (offlineCache == null || !offlineCache.HasValue)
{
var packagePath = _nugetPackageDownloader
.DownloadPackageAsync(new PackageId(packInfo.ResolvedPackageId),
new NuGetVersion(packInfo.Version),
_packageSourceLocation).GetAwaiter().GetResult();
tempFilesToDelete.Add(packagePath);
return (packagePath, packInfo);
}
else
{
_reporter.WriteLine(string.Format(LocalizableStrings.UsingCacheForPackInstall, packInfo.ResolvedPackageId, packInfo.Version, offlineCache));
var packagePath = Path.Combine(offlineCache.Value.Value, $"{packInfo.ResolvedPackageId}.{packInfo.Version}.nupkg");
if (!File.Exists(packagePath))
{
throw new Exception(string.Format(LocalizableStrings.CacheMissingPackage, packInfo.ResolvedPackageId, packInfo.Version, offlineCache));
}
return (packagePath, packInfo);
}
});

foreach (var (packagePath, packInfo) in packagePaths)
{
_reporter.WriteLine(string.Format(LocalizableStrings.InstallingPackVersionMessage, packInfo.ResolvedPackageId, packInfo.Version));
var tempDirsToDelete = new List<string>();
var tempFilesToDelete = new List<string>();
bool shouldRollBackPack = false;

transactionContext.Run(
action: () =>
{
if (PackIsInstalled(packInfo) && !overwriteExistingPacks)
if (!Directory.Exists(Path.GetDirectoryName(packInfo.Path)))
{
Directory.CreateDirectory(Path.GetDirectoryName(packInfo.Path));
}

if (IsSingleFilePack(packInfo))
{
_reporter.WriteLine(string.Format(LocalizableStrings.WorkloadPackAlreadyInstalledMessage, packInfo.ResolvedPackageId, packInfo.Version));
File.Copy(packagePath, packInfo.Path, overwrite: overwriteExistingPacks);
}
else
{
shouldRollBackPack = true;
string packagePath;
if (offlineCache == null || !offlineCache.HasValue)
{
packagePath = _nugetPackageDownloader
.DownloadPackageAsync(new PackageId(packInfo.ResolvedPackageId),
new NuGetVersion(packInfo.Version),
_packageSourceLocation).GetAwaiter().GetResult();
tempFilesToDelete.Add(packagePath);
}
else
{
_reporter.WriteLine(string.Format(LocalizableStrings.UsingCacheForPackInstall, packInfo.ResolvedPackageId, packInfo.Version, offlineCache));
packagePath = Path.Combine(offlineCache.Value.Value, $"{packInfo.ResolvedPackageId}.{packInfo.Version}.nupkg");
if (!File.Exists(packagePath))
{
throw new Exception(string.Format(LocalizableStrings.CacheMissingPackage, packInfo.ResolvedPackageId, packInfo.Version, offlineCache));
}
}
var tempExtractionDir = Path.Combine(_tempPackagesDir.Value, $"{packInfo.ResolvedPackageId}-{packInfo.Version}-extracted");
tempDirsToDelete.Add(tempExtractionDir);

if (!Directory.Exists(Path.GetDirectoryName(packInfo.Path)))
// This directory should have been deleted, but remove it just in case
if (overwriteExistingPacks && Directory.Exists(tempExtractionDir))
{
Directory.CreateDirectory(Path.GetDirectoryName(packInfo.Path));
Directory.Delete(tempExtractionDir, recursive: true);
}

if (IsSingleFilePack(packInfo))
Directory.CreateDirectory(tempExtractionDir);
var packFiles = _nugetPackageDownloader.ExtractPackageAsync(packagePath, new DirectoryPath(tempExtractionDir)).GetAwaiter().GetResult();

if (overwriteExistingPacks && Directory.Exists(packInfo.Path))
{
File.Copy(packagePath, packInfo.Path, overwrite: overwriteExistingPacks);
Directory.Delete(packInfo.Path, recursive: true);
}
else
{
var tempExtractionDir = Path.Combine(_tempPackagesDir.Value, $"{packInfo.ResolvedPackageId}-{packInfo.Version}-extracted");
tempDirsToDelete.Add(tempExtractionDir);

// This directory should have been deleted, but remove it just in case
if (overwriteExistingPacks && Directory.Exists(tempExtractionDir))
{
Directory.Delete(tempExtractionDir, recursive: true);
}

Directory.CreateDirectory(tempExtractionDir);
var packFiles = _nugetPackageDownloader.ExtractPackageAsync(packagePath, new DirectoryPath(tempExtractionDir)).GetAwaiter().GetResult();

if (overwriteExistingPacks && Directory.Exists(packInfo.Path))
{
Directory.Delete(packInfo.Path, recursive: true);
}

FileAccessRetrier.RetryOnMoveAccessFailure(() => DirectoryPath.MoveDirectory(tempExtractionDir, packInfo.Path));
}
}
FileAccessRetrier.RetryOnMoveAccessFailure(() => DirectoryPath.MoveDirectory(tempExtractionDir, packInfo.Path));
}

WritePackInstallationRecord(packInfo, sdkFeatureBand);
},
rollback: () =>
{
try
{
if (shouldRollBackPack)
_reporter.WriteLine(string.Format(LocalizableStrings.RollingBackPackInstall, packInfo.ResolvedPackageId));
DeletePackInstallationRecord(packInfo, sdkFeatureBand);
if (!PackHasInstallRecords(packInfo))
{
_reporter.WriteLine(string.Format(LocalizableStrings.RollingBackPackInstall, packInfo.ResolvedPackageId));
DeletePackInstallationRecord(packInfo, sdkFeatureBand);
if (!PackHasInstallRecords(packInfo))
{
DeletePack(packInfo);
}
DeletePack(packInfo);
}
}
catch (Exception e)
Expand Down
8 changes: 4 additions & 4 deletions test/dotnet-workload-install.Tests/GivenFileBasedWorkloadInstall.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,11 +410,11 @@ public void GivenManagedInstallItCanErrorsWhenMissingOfflineCache()
var version = "6.0.100";
var cachePath = Path.Combine(dotnetRoot, "MockCache");

var exceptionThrown = Assert.Throws<Exception>(() =>
var exceptionThrown = Assert.Throws<AggregateException>(() =>
CliTransaction.RunNew(context => installer.InstallWorkloads(new[] { new WorkloadId("android-sdk-workload") }, new SdkFeatureBand(version), context, new DirectoryPath(cachePath))));
exceptionThrown.Message.Should().Contain(packId);
exceptionThrown.Message.Should().Contain(packVersion);
exceptionThrown.Message.Should().Contain(cachePath);
exceptionThrown.InnerException.Message.Should().Contain(packId);
exceptionThrown.InnerException.Message.Should().Contain(packVersion);
exceptionThrown.InnerException.Message.Should().Contain(cachePath);
}

private (string, FileBasedInstaller, INuGetPackageDownloader, Func<string, IWorkloadResolver>) GetTestInstaller([CallerMemberName] string testName = "", bool failingInstaller = false, string identifier = "", bool manifestDownload = false,
Expand Down
9 changes: 8 additions & 1 deletion test/dotnet-workload-install.Tests/MockNuGetPackageInstaller.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,14 @@ public Task<string> DownloadPackageAsync(PackageId packageId,
DownloadCallResult.Add(path);
if (_downloadPath != string.Empty)
{
File.WriteAllText(path, string.Empty);
try
{
File.WriteAllText(path, string.Empty);
}
catch (IOException)
{
// Do not write this file twice in parallel
}
}
_lastPackageVersion = packageVersion ?? new NuGetVersion("1.0.42");
return Task.FromResult(path);
Expand Down
Loading