C#: Automatically use configured private registry feeds

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs

@@ -1,14 +1,22 @@
 using System;
-using System.Diagnostics;
+using System.Collections.Generic;
 using System.IO;
 using System.Security.Cryptography.X509Certificates;
 using Semmle.Util;
 using Semmle.Util.Logging;
+using Newtonsoft.Json;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     public class DependabotProxy : IDisposable
     {
+        /// <summary>
+        /// Represents configurations for package registries.
+        /// </summary>
+        /// <param name="Type">The type of package registry.</param>
+        /// <param name="URL">The URL of the package registry.</param>
+        public record class RegistryConfig(string Type, string URL);
+
         private readonly string host;
         private readonly string port;
 
@@ -17,6 +25,10 @@ public class DependabotProxy : IDisposable
         /// </summary>
         internal string Address { get; }
         /// <summary>
+        /// The URLs of package registries that are configured for the proxy.
+        /// </summary>
+        internal HashSet<string> RegistryURLs { get; }
+        /// <summary>
         /// The path to the temporary file where the certificate is stored.
         /// </summary>
         internal string? CertificatePath { get; private set; }
@@ -67,6 +79,39 @@ public class DependabotProxy : IDisposable
                 result.Certificate = X509Certificate2.CreateFromPem(cert);
             }
 
+            // Try to obtain the list of private registry URLs.
+            var registryURLs = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyURLs);
+
+            if (!string.IsNullOrWhiteSpace(registryURLs))
+            {
+                try
+                {
+                    // The value of the environment variable should be a JSON array of objects, such as:
+                    // [ { "type": "nuget_feed", "url": "https://nuget.pkg.github.com/org/index.json" } ]
+                    var array = JsonConvert.DeserializeObject<List<RegistryConfig>>(registryURLs);
+                    if (array is not null)
+                    {
+                        foreach (RegistryConfig config in array)
+                        {
+                            // The array contains all configured private registries, not just ones for C#.
+                            // We ignore the non-C# ones here.
+                            if (!config.Type.Equals("nuget_feed"))
+                            {
+                                logger.LogDebug($"Ignoring registry at '{config.URL}' since it is not of type 'nuget_feed'.");
+                                continue;
+                            }
+
+                            logger.LogInfo($"Found private registry at '{config.URL}'");
+                            result.RegistryURLs.Add(config.URL);
+                        }
+                    }
+                }
+                catch (JsonException ex)
+                {
+                    logger.LogError($"Unable to parse '{EnvironmentVariableNames.ProxyURLs}': {ex.Message}");
+                }
+            }
+
             return result;
         }
 
@@ -75,6 +120,7 @@ private DependabotProxy(string host, string port)
             this.host = host;
             this.port = port;
             this.Address = $"http://{this.host}:{this.port}";
+            this.RegistryURLs = new HashSet<string>();
         }
 
         public void Dispose()

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-
+using System.Text;
 using Newtonsoft.Json.Linq;
 
 using Semmle.Util;
@@ -77,6 +77,11 @@ private string GetRestoreArgs(RestoreSettings restoreSettings)
                 args += " /p:EnableWindowsTargeting=true";
             }
 
+            if (restoreSettings.ExtraArgs != null)
+            {
+                args += $" {restoreSettings.ExtraArgs}";
+            }
+
             return args;
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -89,5 +89,10 @@ internal static class EnvironmentVariableNames
         /// Contains the certificate used by the Dependabot proxy.
         /// </summary>
         public const string ProxyCertificate = "CODEQL_PROXY_CA_CERTIFICATE";
+
+        /// <summary>
+        /// Contains the URLs of private nuget registries as a JSON array.
+        /// </summary>
+        public const string ProxyURLs = "CODEQL_PROXY_URLS";
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -17,7 +17,7 @@ public interface IDotNet
         IList<string> GetNugetFeedsFromFolder(string folderPath);
     }
 
-    public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false, bool TargetWindows = false);
+    public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? ExtraArgs = null, string? PathToNugetConfig = null, bool ForceReevaluation = false, bool TargetWindows = false);
 
     public partial record class RestoreResult(bool Success, IList<string> Output)
     {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -156,7 +156,7 @@ public HashSet<AssemblyLookupLocation> Restore()
 
             var restoredProjects = RestoreSolutions(out var container);
             var projects = fileProvider.Projects.Except(restoredProjects);
-            RestoreProjects(projects, out var containers);
+            RestoreProjects(projects, explicitFeeds, out var containers);
 
             var dependencies = containers.Flatten(container);
 
@@ -260,8 +260,34 @@ private IEnumerable<string> RestoreSolutions(out DependencyContainer dependencie
         /// Populates dependencies with the relative paths to the assets files generated by the restore.
         /// </summary>
         /// <param name="projects">A list of paths to project files.</param>
-        private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<DependencyContainer> dependencies)
+        private void RestoreProjects(IEnumerable<string> projects, HashSet<string>? configuredSources, out ConcurrentBag<DependencyContainer> dependencies)
         {
+            // Conservatively, we only set this to a non-null value if a Dependabot proxy is enabled.
+            // This ensures that we continue to get the old behaviour where feeds are taken from
+            // `nuget.config` files instead of the command-line arguments.
+            string? extraArgs = null;
+
+            if (this.dependabotProxy is not null)
+            {
+                // If the Dependabot proxy is configured, then our main goal is to make `dotnet` aware
+                // of the private registry feeds. However, since providing them as command-line arguments
+                // to `dotnet` ignores other feeds that may be configured, we also need to add the feeds
+                // we have discovered from analysing `nuget.config` files.
+                var sources = configuredSources ?? new();
+                sources.Add(PublicNugetOrgFeed);
+                this.dependabotProxy.RegistryURLs.ForEach(url => sources.Add(url));
+
+                // Add package sources. If any are present, they override all sources specified in
+                // the configuration file(s).
+                var feedArgs = new StringBuilder();
+                foreach (string source in sources)
+                {
+                    feedArgs.Append($" -s {source}");
+                }
+
+                extraArgs = feedArgs.ToString();
+            }
+
             var successCount = 0;
             var nugetSourceFailures = 0;
             ConcurrentBag<DependencyContainer> collectedDependencies = [];
@@ -276,7 +302,7 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
                 foreach (var project in projectGroup)
                 {
                     logger.LogInfo($"Restoring project {project}...");
-                    var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, TargetWindows: isWindows));
+                    var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, extraArgs, TargetWindows: isWindows));
                     assets.AddDependenciesRange(res.AssetsFilePaths);
                     lock (sync)
                     {
@@ -674,10 +700,40 @@ private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount,
             return (timeoutMilliSeconds, tryCount);
         }
 
+        /// <summary>
+        /// Checks that we can connect to all Nuget feeds that are explicitly configured in configuration files.
+        /// </summary>
+        /// <param name="explicitFeeds">Outputs the set of explicit feeds.</param>
+        /// <returns>True if all feeds are reachable or false otherwise.</returns>
         private bool CheckFeeds(out HashSet<string> explicitFeeds)
         {
-            logger.LogInfo("Checking Nuget feeds...");
             (explicitFeeds, var allFeeds) = GetAllFeeds();
+            HashSet<string> feedsToCheck = explicitFeeds;
+
+            // If private package registries are configured for C#, then check those
+            // in addition to the ones that are configured in `nuget.config` files.
+            this.dependabotProxy?.RegistryURLs.ForEach(url => feedsToCheck.Add(url));
+
+            var allFeedsReachable = this.CheckFeeds(feedsToCheck);
+
+            var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet();
+            if (inheritedFeeds.Count > 0)
+            {
+                logger.LogInfo($"Inherited Nuget feeds (not checked for reachability): {string.Join(", ", inheritedFeeds.OrderBy(f => f))}");
+                compilationInfoContainer.CompilationInfos.Add(("Inherited Nuget feed count", inheritedFeeds.Count.ToString()));
+            }
+
+            return allFeedsReachable;
+        }
+
+        /// <summary>
+        /// Checks that we can connect to the specified Nuget feeds.
+        /// </summary>
+        /// <param name="feeds">The set of package feeds to check.</param>
+        /// <returns>True if all feeds are reachable or false otherwise.</returns>
+        private bool CheckFeeds(HashSet<string> feeds)
+        {
+            logger.LogInfo("Checking that Nuget feeds are reachable...");
 
             var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
                 .ToHashSet();
@@ -689,7 +745,7 @@ private bool CheckFeeds(out HashSet<string> explicitFeeds)
 
             var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
 
-            var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed, initialTimeout, tryCount));
+            var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed, initialTimeout, tryCount));
             if (!allFeedsReachable)
             {
                 logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.");
@@ -704,14 +760,6 @@ private bool CheckFeeds(out HashSet<string> explicitFeeds)
             }
             compilationInfoContainer.CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
 
-
-            var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet();
-            if (inheritedFeeds.Count > 0)
-            {
-                logger.LogInfo($"Inherited Nuget feeds (not checked for reachability): {string.Join(", ", inheritedFeeds.OrderBy(f => f))}");
-                compilationInfoContainer.CompilationInfos.Add(("Inherited Nuget feed count", inheritedFeeds.Count.ToString()));
-            }
-
             return allFeedsReachable;
         }
 

csharp/extractor/Semmle.Extraction.Tests/DotNet.cs

@@ -123,7 +123,7 @@ public void TestDotnetRestoreProjectToDirectory2()
             var dotnet = MakeDotnet(dotnetCliInvoker);
 
             // Execute
-            var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config"));
+            var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, null, "myconfig.config"));
 
             // Verify
             var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -141,7 +141,7 @@ public void TestDotnetRestoreProjectToDirectory3()
             var dotnet = MakeDotnet(dotnetCliInvoker);
 
             // Execute
-            var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config", true));
+            var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, null, "myconfig.config", true));
 
             // Verify
             var lastArgs = dotnetCliInvoker.GetLastArgs();