From 9cfa2c44e4f42dd6c061c05f4f341adf79f666f0 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 24 Jun 2026 10:56:07 +0200 Subject: [PATCH 01/12] C#: Re-factor GetAllFeeds into properties in the Feed Manager and encapsulate as immutable hash sets. --- .../FeedManager.cs | 26 +++++++++++++++---- .../NugetPackageRestorer.cs | 7 ++--- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index 744b60f3d3f5..1dc4c2ac1666 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -29,6 +29,12 @@ internal sealed partial class FeedManager : IDisposable public bool HasPrivateRegistryFeeds { get; } public bool CheckNugetFeedResponsiveness { get; } = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.CheckNugetFeedResponsiveness); + private readonly Lazy> lazyExplicitFeeds; + public ImmutableHashSet ExplicitFeeds => lazyExplicitFeeds.Value; + + private readonly Lazy> lazyAllFeeds; + public ImmutableHashSet AllFeeds => lazyAllFeeds.Value; + public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotProxy, FileProvider fileProvider) { this.logger = logger; @@ -38,6 +44,9 @@ public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotPr PrivateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? []; HasPrivateRegistryFeeds = PrivateRegistryFeeds.Count > 0; emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger); + + lazyExplicitFeeds = new Lazy>(GetExplicitFeeds); + lazyAllFeeds = new Lazy>(GetAllFeeds); } private string? GetDirectoryName(string path) @@ -271,7 +280,7 @@ private HashSet GetExcludedFeeds() /// True if there is a timeout when trying to reach the feeds (excluding any feeds that are configured /// to be excluded from the check) or false otherwise. /// - public bool CheckSpecifiedFeeds(HashSet feeds, out HashSet reachableFeeds) + public bool CheckSpecifiedFeeds(ImmutableHashSet feeds, out HashSet reachableFeeds) { // Exclude any feeds from the feed check that are configured by the corresponding environment variable. // These feeds are always assumed to be reachable. @@ -342,7 +351,7 @@ private List GetReachableNuGetFeeds(HashSet feedsToCheck, bool i return reachableFeeds; } - public List GetReachableFallbackNugetFeeds(HashSet? feedsFromNugetConfigs) + public List GetReachableFallbackNugetFeeds(ImmutableHashSet? feedsFromNugetConfigs) { var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet(); if (fallbackFeeds.Count == 0) @@ -365,7 +374,7 @@ public List GetReachableFallbackNugetFeeds(HashSet? feedsFromNug return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true, out var _); } - public (HashSet explicitFeeds, HashSet allFeeds) GetAllFeeds() + private ImmutableHashSet GetExplicitFeeds() { var nugetConfigs = fileProvider.NugetConfigs; @@ -391,10 +400,17 @@ public List GetReachableFallbackNugetFeeds(HashSet? feedsFromNug explicitFeeds.UnionWith(PrivateRegistryFeeds); } + return explicitFeeds.ToImmutableHashSet(); + } + + private ImmutableHashSet GetAllFeeds() + { + var nugetConfigs = fileProvider.NugetConfigs; + HashSet allFeeds = []; // Add all explicitFeeds to the set of all feeds. - allFeeds.UnionWith(explicitFeeds); + allFeeds.UnionWith(ExplicitFeeds); // Obtain the list of feeds from the root source directory. // If a NuGet file is present it will be respected, otherwise we will just get the machine/environment specific feeds. @@ -414,7 +430,7 @@ public List GetReachableFallbackNugetFeeds(HashSet? feedsFromNug logger.LogInfo($"Found {allFeeds.Count} NuGet feeds (with inherited ones) in nuget.config files: {string.Join(", ", allFeeds.OrderBy(f => f))}"); - return (explicitFeeds, allFeeds); + return allFeeds.ToImmutableHashSet(); } [GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 9da2018dffbc..2b55923e46b3 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -117,11 +117,12 @@ public HashSet Restore() // Find feeds that are configured in NuGet.config files and divide them into ones that // are explicitly configured for the project or by a private registry, and "all feeds" // (including inherited ones) from other locations on the host outside of the working directory. - (var explicitFeeds, var allFeeds) = feedManager.GetAllFeeds(); + var explicitFeeds = feedManager.ExplicitFeeds; + var allFeeds = feedManager.AllFeeds; if (feedManager.CheckNugetFeedResponsiveness) { - var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet(); + var inheritedFeeds = allFeeds.Except(explicitFeeds).ToImmutableHashSet(); if (inheritedFeeds.Count > 0) { @@ -312,7 +313,7 @@ private void RestoreProjects(IEnumerable projects, HashSet reach compilationInfoContainer.CompilationInfos.Add(("Failed project restore with missing package error", nugetMissingPackageFailures.ToString())); } - private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(IEnumerable usedPackageNames, HashSet? feedsFromNugetConfigs) + private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(IEnumerable usedPackageNames, ImmutableHashSet? feedsFromNugetConfigs) { var reachableFallbackFeeds = feedManager.GetReachableFallbackNugetFeeds(feedsFromNugetConfigs); compilationInfoContainer.CompilationInfos.Add(("Reachable fallback NuGet feed count", reachableFallbackFeeds.Count.ToString())); From 11372ca00798f025e1960316bd198c52bcc8352a Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Wed, 24 Jun 2026 12:56:36 +0200 Subject: [PATCH 02/12] C#: Re-factor more feed sets into the feed manager (and further use immutable hash sets). --- .../FeedManager.cs | 70 ++++++++++++++++--- .../NugetPackageRestorer.cs | 20 ++---- .../PackagesConfigRestorer.cs | 7 +- 3 files changed, 70 insertions(+), 27 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index 1dc4c2ac1666..b0a402148220 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -24,31 +24,79 @@ internal sealed partial class FeedManager : IDisposable private readonly FileProvider fileProvider; private readonly DependabotProxy? dependabotProxy; private readonly DependencyDirectory emptyPackageDirectory; + private readonly ImmutableHashSet privateRegistryFeeds; - public ImmutableHashSet PrivateRegistryFeeds { get; } public bool HasPrivateRegistryFeeds { get; } public bool CheckNugetFeedResponsiveness { get; } = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.CheckNugetFeedResponsiveness); private readonly Lazy> lazyExplicitFeeds; + + /// + /// Gets the list of NuGet feeds that are explicitly configured + /// - NuGet configuration files. + /// - Private package registries that are configured for C#. + /// public ImmutableHashSet ExplicitFeeds => lazyExplicitFeeds.Value; private readonly Lazy> lazyAllFeeds; + + /// + /// Gets the list of all NuGet feeds that are configured in the environment. That is + /// - Explicit feeds + /// - Inherited feeds from the machine and environment (if not explicitly disabled by a + /// root directory NuGet configuration). + /// public ImmutableHashSet AllFeeds => lazyAllFeeds.Value; + /// + /// Gets the list of inherited NuGet feeds that are configured in the environment. + /// + public ImmutableHashSet InheritedFeeds => AllFeeds.Except(ExplicitFeeds).ToImmutableHashSet(); + + private readonly Lazy<(bool, ImmutableHashSet)> lazyReachableExplicitFeeds; + + /// + /// Gets whether there was a timeout when checking the reachability of the explicitly configured NuGet feeds. + /// + public bool ExplicitFeedTimeout => lazyReachableExplicitFeeds.Value.Item1; + + /// + /// Gets the list of reachable NuGet feeds that are explicitly configured. + /// + public ImmutableHashSet ReachableExplicitFeeds => lazyReachableExplicitFeeds.Value.Item2; + + private readonly Lazy> lazyReachableFeeds; + /// + /// Gets the list of reachable NuGet feeds that are configured in the environment. + /// + public ImmutableHashSet ReachableFeeds => lazyReachableFeeds.Value; + public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotProxy, FileProvider fileProvider) { this.logger = logger; this.dotnet = dotnet; this.dependabotProxy = dependabotProxy; this.fileProvider = fileProvider; - PrivateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? []; - HasPrivateRegistryFeeds = PrivateRegistryFeeds.Count > 0; + privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? []; + HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0; emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger); lazyExplicitFeeds = new Lazy>(GetExplicitFeeds); lazyAllFeeds = new Lazy>(GetAllFeeds); + lazyReachableExplicitFeeds = new Lazy<(bool, ImmutableHashSet)>(() => + { + var timeout = CheckSpecifiedFeeds(ExplicitFeeds, out var reachableFeeds); + return (timeout, reachableFeeds); + }); + lazyReachableFeeds = new Lazy>(() => + { + // Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific). + CheckSpecifiedFeeds(InheritedFeeds, out var reachableInheritedFeeds); + return ReachableExplicitFeeds.Union(reachableInheritedFeeds).ToImmutableHashSet(); + }); } + private string? GetDirectoryName(string path) { try @@ -124,7 +172,7 @@ public string FeedsToRestoreArgument(IEnumerable feeds, string sourceArg /// Path to project/solution/packages.config /// The set of reachable NuGet feeds. /// The list of NuGet feeds to use for this restore. - public IEnumerable FeedsToUse(string path, HashSet reachableFeeds) + public IEnumerable FeedsToUse(string path, ImmutableHashSet reachableFeeds) { // Find the path specific feeds. var folder = GetDirectoryName(path); @@ -132,7 +180,7 @@ public IEnumerable FeedsToUse(string path, HashSet reachableFeed if (HasPrivateRegistryFeeds) { - feedsToConsider.UnionWith(PrivateRegistryFeeds); + feedsToConsider.UnionWith(privateRegistryFeeds); } var feedsToUse = CheckNugetFeedResponsiveness @@ -150,7 +198,7 @@ public IEnumerable FeedsToUse(string path, HashSet reachableFeed /// Path to project/solution /// The set of reachable NuGet feeds. /// A string representing the NuGet sources argument for the restore command. - public string? MakeDotnetRestoreSourcesArgument(string path, HashSet reachableFeeds) + public string? MakeDotnetRestoreSourcesArgument(string path, ImmutableHashSet reachableFeeds) { // Do not construct a set of explicit NuGet sources to use for restore. if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds) @@ -280,7 +328,7 @@ private HashSet GetExcludedFeeds() /// True if there is a timeout when trying to reach the feeds (excluding any feeds that are configured /// to be excluded from the check) or false otherwise. /// - public bool CheckSpecifiedFeeds(ImmutableHashSet feeds, out HashSet reachableFeeds) + private bool CheckSpecifiedFeeds(ImmutableHashSet feeds, out ImmutableHashSet reachableFeeds) { // Exclude any feeds from the feed check that are configured by the corresponding environment variable. // These feeds are always assumed to be reachable. @@ -296,10 +344,10 @@ public bool CheckSpecifiedFeeds(ImmutableHashSet feeds, out HashSet excludedFeeds.Contains(feed))); + reachableFeeds = reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet(); return isTimeout; } @@ -396,8 +444,8 @@ private ImmutableHashSet GetExplicitFeeds() // in addition to the ones that are configured in `nuget.config` files. if (HasPrivateRegistryFeeds) { - logger.LogInfo($"Found {PrivateRegistryFeeds.Count} private registry feeds configured for C#: {string.Join(", ", PrivateRegistryFeeds.OrderBy(f => f))}"); - explicitFeeds.UnionWith(PrivateRegistryFeeds); + logger.LogInfo($"Found {privateRegistryFeeds.Count} private registry feeds configured for C#: {string.Join(", ", privateRegistryFeeds.OrderBy(f => f))}"); + explicitFeeds.UnionWith(privateRegistryFeeds); } return explicitFeeds.ToImmutableHashSet(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 2b55923e46b3..98d5394b9235 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -110,7 +110,7 @@ public HashSet Restore() logger.LogInfo($"Checking NuGet feed responsiveness: {feedManager.CheckNugetFeedResponsiveness}"); compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", feedManager.CheckNugetFeedResponsiveness ? "1" : "0")); - HashSet reachableFeeds = []; + ImmutableHashSet reachableFeeds = []; EmitNugetConfigDiagnostics(); @@ -118,24 +118,20 @@ public HashSet Restore() // are explicitly configured for the project or by a private registry, and "all feeds" // (including inherited ones) from other locations on the host outside of the working directory. var explicitFeeds = feedManager.ExplicitFeeds; - var allFeeds = feedManager.AllFeeds; if (feedManager.CheckNugetFeedResponsiveness) { - var inheritedFeeds = allFeeds.Except(explicitFeeds).ToImmutableHashSet(); + var inheritedFeeds = feedManager.InheritedFeeds; if (inheritedFeeds.Count > 0) { compilationInfoContainer.CompilationInfos.Add(("Inherited NuGet feed count", inheritedFeeds.Count.ToString())); } - var timeout = feedManager.CheckSpecifiedFeeds(explicitFeeds, out var reachableExplicitFeeds); - reachableFeeds.UnionWith(reachableExplicitFeeds); - - var allExplicitReachable = explicitFeeds.Count == reachableExplicitFeeds.Count; + var allExplicitReachable = explicitFeeds.Count == feedManager.ReachableExplicitFeeds.Count; EmitUnreachableFeedsDiagnostics(allExplicitReachable); - if (timeout) + if (feedManager.ExplicitFeedTimeout) { // If we experience a timeout, we use this fallback. // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too. @@ -145,9 +141,7 @@ public HashSet Restore() : [unresponsiveMissingPackageLocation]; } - // Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific). - feedManager.CheckSpecifiedFeeds(inheritedFeeds, out var reachableInheritedFeeds); - reachableFeeds.UnionWith(reachableInheritedFeeds); + reachableFeeds = feedManager.ReachableFeeds; } try @@ -224,7 +218,7 @@ public HashSet Restore() /// Populates dependencies with the relevant dependencies from the assets files generated by the restore. /// Returns a list of projects that are up to date with respect to restore. /// - private IEnumerable RestoreSolutions(HashSet reachableFeeds, out DependencyContainer dependencies) + private IEnumerable RestoreSolutions(ImmutableHashSet reachableFeeds, out DependencyContainer dependencies) { var successCount = 0; var nugetSourceFailures = 0; @@ -269,7 +263,7 @@ private IEnumerable RestoreSolutions(HashSet reachableFeeds, out /// /// A list of paths to project files. /// The set of reachable NuGet feeds. - private void RestoreProjects(IEnumerable projects, HashSet reachableFeeds, out ConcurrentBag dependencies) + private void RestoreProjects(IEnumerable projects, ImmutableHashSet reachableFeeds, out ConcurrentBag dependencies) { var successCount = 0; var nugetSourceFailures = 0; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs index af484ba406e4..26f9e42d435d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Diagnostics; using System.IO; using System.Linq; @@ -33,7 +34,7 @@ internal interface IPackagesConfigRestore /// internal class PackagesConfigRestoreFactory { - public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, HashSet reachableFeeds) + public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, ImmutableHashSet reachableFeeds) { if (SystemBuildActions.Instance.IsWindows() || SystemBuildActions.Instance.IsMonoInstalled()) { @@ -64,7 +65,7 @@ private class NugetExeWrapper : IPackagesConfigRestore /// private readonly DependencyDirectory packageDirectory; private readonly FeedManager feedManager; - private readonly HashSet reachableFeeds; + private readonly ImmutableHashSet reachableFeeds; private bool IsWindows => SystemBuildActions.Instance.IsWindows(); @@ -75,7 +76,7 @@ private class NugetExeWrapper : IPackagesConfigRestore /// /// Create the package manager for a specified source tree. /// - public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, HashSet reachableFeeds) + public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, ImmutableHashSet reachableFeeds) { this.fileProvider = fileProvider; this.packageDirectory = packageDirectory; From c2d4be52b774ebb269e9242f6f254c7407deeb46 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Thu, 25 Jun 2026 14:16:50 +0200 Subject: [PATCH 03/12] C#: Do not pass around the set of reachable feeds. This implicitly delays the calculation of the reachable feeds until it is needed/used. --- .../FeedManager.cs | 10 ++++------ .../NugetPackageRestorer.cs | 18 +++++++----------- .../PackagesConfigRestorer.cs | 10 ++++------ 3 files changed, 15 insertions(+), 23 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index b0a402148220..ec5b781eaacf 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -170,9 +170,8 @@ public string FeedsToRestoreArgument(IEnumerable feeds, string sourceArg /// (2) Use private registries, if they are configured /// /// Path to project/solution/packages.config - /// The set of reachable NuGet feeds. /// The list of NuGet feeds to use for this restore. - public IEnumerable FeedsToUse(string path, ImmutableHashSet reachableFeeds) + public IEnumerable FeedsToUse(string path) { // Find the path specific feeds. var folder = GetDirectoryName(path); @@ -184,7 +183,7 @@ public IEnumerable FeedsToUse(string path, ImmutableHashSet reac } var feedsToUse = CheckNugetFeedResponsiveness - ? feedsToConsider.Where(reachableFeeds.Contains) + ? feedsToConsider.Where(ReachableFeeds.Contains) : feedsToConsider; return feedsToUse; @@ -196,9 +195,8 @@ public IEnumerable FeedsToUse(string path, ImmutableHashSet reac /// (2) Use private registries, if they are configured /// /// Path to project/solution - /// The set of reachable NuGet feeds. /// A string representing the NuGet sources argument for the restore command. - public string? MakeDotnetRestoreSourcesArgument(string path, ImmutableHashSet reachableFeeds) + public string? MakeDotnetRestoreSourcesArgument(string path) { // Do not construct a set of explicit NuGet sources to use for restore. if (!CheckNugetFeedResponsiveness && !HasPrivateRegistryFeeds) @@ -206,7 +204,7 @@ public IEnumerable FeedsToUse(string path, ImmutableHashSet reac return null; } - var feedsToUse = FeedsToUse(path, reachableFeeds); + var feedsToUse = FeedsToUse(path); return FeedsToRestoreArgument(feedsToUse, "-s"); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 98d5394b9235..bcccfda829a8 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -110,8 +110,6 @@ public HashSet Restore() logger.LogInfo($"Checking NuGet feed responsiveness: {feedManager.CheckNugetFeedResponsiveness}"); compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", feedManager.CheckNugetFeedResponsiveness ? "1" : "0")); - ImmutableHashSet reachableFeeds = []; - EmitNugetConfigDiagnostics(); // Find feeds that are configured in NuGet.config files and divide them into ones that @@ -141,12 +139,11 @@ public HashSet Restore() : [unresponsiveMissingPackageLocation]; } - reachableFeeds = feedManager.ReachableFeeds; } try { - var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, feedManager, reachableFeeds); + var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, feedManager); var count = packagesConfigRestore.InstallPackages(); if (packagesConfigRestore.PackageCount > 0) { @@ -184,9 +181,9 @@ public HashSet Restore() } // Restore project dependencies with `dotnet restore`. - var restoredProjects = RestoreSolutions(reachableFeeds, out var container); + var restoredProjects = RestoreSolutions(out var container); var projects = fileProvider.Projects.Except(restoredProjects); - RestoreProjects(projects, reachableFeeds, out var containers); + RestoreProjects(projects, out var containers); var dependencies = containers.Flatten(container); @@ -218,7 +215,7 @@ public HashSet Restore() /// Populates dependencies with the relevant dependencies from the assets files generated by the restore. /// Returns a list of projects that are up to date with respect to restore. /// - private IEnumerable RestoreSolutions(ImmutableHashSet reachableFeeds, out DependencyContainer dependencies) + private IEnumerable RestoreSolutions(out DependencyContainer dependencies) { var successCount = 0; var nugetSourceFailures = 0; @@ -231,7 +228,7 @@ private IEnumerable RestoreSolutions(ImmutableHashSet reachableF var projects = fileProvider.Solutions.SelectMany(solution => { logger.LogInfo($"Restoring solution {solution}..."); - var nugetSources = feedManager.MakeDotnetRestoreSourcesArgument(solution, reachableFeeds); + var nugetSources = feedManager.MakeDotnetRestoreSourcesArgument(solution); var res = dotnet.Restore(new(solution, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, NugetSources: nugetSources, TargetWindows: isWindows)); if (res.Success) { @@ -262,8 +259,7 @@ private IEnumerable RestoreSolutions(ImmutableHashSet reachableF /// Populates dependencies with the relative paths to the assets files generated by the restore. /// /// A list of paths to project files. - /// The set of reachable NuGet feeds. - private void RestoreProjects(IEnumerable projects, ImmutableHashSet reachableFeeds, out ConcurrentBag dependencies) + private void RestoreProjects(IEnumerable projects, out ConcurrentBag dependencies) { var successCount = 0; var nugetSourceFailures = 0; @@ -280,7 +276,7 @@ private void RestoreProjects(IEnumerable projects, ImmutableHashSet internal class PackagesConfigRestoreFactory { - public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, ImmutableHashSet reachableFeeds) + public static IPackagesConfigRestore Create(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager) { if (SystemBuildActions.Instance.IsWindows() || SystemBuildActions.Instance.IsMonoInstalled()) { - return new NugetExeWrapper(fileProvider, packageDirectory, logger, feedManager, reachableFeeds); + return new NugetExeWrapper(fileProvider, packageDirectory, logger, feedManager); } return new NoOpPackagesConfig(fileProvider.PackagesConfigs, logger); @@ -65,7 +65,6 @@ private class NugetExeWrapper : IPackagesConfigRestore /// private readonly DependencyDirectory packageDirectory; private readonly FeedManager feedManager; - private readonly ImmutableHashSet reachableFeeds; private bool IsWindows => SystemBuildActions.Instance.IsWindows(); @@ -76,13 +75,12 @@ private class NugetExeWrapper : IPackagesConfigRestore /// /// Create the package manager for a specified source tree. /// - public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager, ImmutableHashSet reachableFeeds) + public NugetExeWrapper(FileProvider fileProvider, DependencyDirectory packageDirectory, Semmle.Util.Logging.ILogger logger, FeedManager feedManager) { this.fileProvider = fileProvider; this.packageDirectory = packageDirectory; this.logger = logger; this.feedManager = feedManager; - this.reachableFeeds = reachableFeeds; if (fileProvider.PackagesConfigs.Count > 0) { @@ -171,7 +169,7 @@ private bool TryRestoreNugetPackage(string packagesConfig) logger.LogInfo($"Restoring file \"{packagesConfig}\"..."); var sourcesArgument = ""; - var feedsToUse = feedManager.FeedsToUse(packagesConfig, reachableFeeds).ToList(); + var feedsToUse = feedManager.FeedsToUse(packagesConfig).ToList(); var useDefaultFeed = feedsToUse.Count == 0 && IsDefaultFeedReachable; // Explicitly construct the sources to be used for the restore command when checking feed From 3151f1366f189c293cb946feff48d94859958277 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Thu, 25 Jun 2026 15:21:47 +0200 Subject: [PATCH 04/12] C#: Use feed management for more package download flows. --- .../FeedManager.cs | 35 ++++++++---- .../NugetPackageRestorer.cs | 53 +++++++------------ 2 files changed, 46 insertions(+), 42 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index ec5b781eaacf..99568fa9ce19 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -164,6 +164,20 @@ public string FeedsToRestoreArgument(IEnumerable feeds, string sourceArg return feedArgs.ToString(); } + private IEnumerable FeedsToUseAux(HashSet feedsToConsider) + { + if (HasPrivateRegistryFeeds) + { + feedsToConsider.UnionWith(privateRegistryFeeds); + } + + var feedsToUse = CheckNugetFeedResponsiveness + ? feedsToConsider.Where(ReachableFeeds.Contains) + : feedsToConsider; + + return feedsToUse; + } + /// /// Constructs the list of NuGet sources to use for this restore. /// (1) Use the feeds we get from `dotnet nuget list source` @@ -177,16 +191,19 @@ public IEnumerable FeedsToUse(string path) var folder = GetDirectoryName(path); var feedsToConsider = folder is not null ? GetFeedsFromFolder(folder).ToHashSet() : new HashSet(); - if (HasPrivateRegistryFeeds) - { - feedsToConsider.UnionWith(privateRegistryFeeds); - } + return FeedsToUseAux(feedsToConsider); + } - var feedsToUse = CheckNugetFeedResponsiveness - ? feedsToConsider.Where(ReachableFeeds.Contains) - : feedsToConsider; + public IEnumerable FeedsToUseFromConfig(string config) + { + var feedsToConsider = GetFeedsFromNugetConfig(config).ToHashSet(); - return feedsToUse; + return FeedsToUseAux(feedsToConsider); + } + + public string FeedsToDotnetRestoreArgument(IEnumerable feeds) + { + return FeedsToRestoreArgument(feeds, "-s"); } /// @@ -206,7 +223,7 @@ public IEnumerable FeedsToUse(string path) var feedsToUse = FeedsToUse(path); - return FeedsToRestoreArgument(feedsToUse, "-s"); + return FeedsToDotnetRestoreArgument(feedsToUse); } private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index bcccfda829a8..8dd04e989edd 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -350,10 +350,18 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag feeds = []; + if (fallbackNugetFeeds is not null) + { + feeds = fallbackNugetFeeds; + } + else if (GetNugetConfig() is string config) + { + feeds = feedManager.FeedsToUseFromConfig(config); + } + + var nugetSources = feedManager.FeedsToDotnetRestoreArgument(feeds); compilationInfoContainer.CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString())); @@ -362,7 +370,7 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag { - var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource, tryWithoutNugetConfig: fallbackNugetFeeds is null); + var success = TryRestorePackageManually(package.Name, nugetSources, package.PackageReferenceSource, tryWithoutNugetConfig: fallbackNugetFeeds is null); if (!success) { return; @@ -379,27 +387,6 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag fallbackNugetFeeds, string folderPath) - { - var sb = new StringBuilder(); - fallbackNugetFeeds.ForEach((feed, index) => sb.AppendLine($"")); - - var nugetConfigPath = Path.Join(folderPath, "nuget.config"); - logger.LogInfo($"Creating fallback nuget.config file {nugetConfigPath}."); - File.WriteAllText(nugetConfigPath, - $""" - - - - - {sb} - - - """); - - return nugetConfigPath; - } - private string? GetNugetConfig() { var nugetConfigs = fileProvider.NugetConfigs; @@ -489,7 +476,7 @@ private static IEnumerable GetRestoredPackageDirectoryNames(DirectoryInf .Select(d => Path.GetFileName(d).ToLowerInvariant()); } - private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, + private bool TryRestorePackageManually(string package, string? nugetSources = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryWithoutNugetConfig = true, bool tryPrereleaseVersion = true) { logger.LogInfo($"Restoring package {package}..."); @@ -512,17 +499,17 @@ private bool TryRestorePackageManually(string package, string? nugetConfig = nul return false; } - var res = TryRestorePackageManually(package, nugetConfig, tempDir, tryPrereleaseVersion); + var res = TryRestorePackageManually(package, nugetSources, tempDir, tryPrereleaseVersion); if (res.Success) { return true; } - if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null) + if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetSources is not null && !feedManager.CheckNugetFeedResponsiveness) { logger.LogDebug($"Trying to restore '{package}' without nuget.config."); // Restore could not be completed because the listed source is unavailable. Try without the nuget.config: - res = TryRestorePackageManually(package, nugetConfig: null, tempDir, tryPrereleaseVersion); + res = TryRestorePackageManually(package, nugetSources: null, tempDir, tryPrereleaseVersion); if (res.Success) { return true; @@ -533,16 +520,16 @@ private bool TryRestorePackageManually(string package, string? nugetConfig = nul return false; } - private RestoreResult TryRestorePackageManually(string package, string? nugetConfig, TemporaryDirectory tempDir, bool tryPrereleaseVersion) + private RestoreResult TryRestorePackageManually(string package, string? nugetSources, TemporaryDirectory tempDir, bool tryPrereleaseVersion) { - var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true)); + var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, NugetSources: nugetSources, ForceReevaluation: true)); if (!res.Success && tryPrereleaseVersion && res.HasNugetNoStablePackageVersionError) { logger.LogDebug($"Failed to restore nuget package {package} because no stable version was found."); TryChangePackageVersion(tempDir.DirInfo, "*-*"); - res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true)); + res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, NugetSources: nugetSources, ForceReevaluation: true)); if (!res.Success) { TryChangePackageVersion(tempDir.DirInfo, "*"); From e7551af991e56d94537029ad019b7ff91b002abf Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Thu, 25 Jun 2026 15:31:17 +0200 Subject: [PATCH 05/12] C#: Remove the dotnet restore config file option. --- .../Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs | 5 ----- .../IDotNet.cs | 2 +- csharp/extractor/Semmle.Extraction.Tests/DotNet.cs | 8 ++++---- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index e93a29336126..e02d157dc620 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -80,11 +80,6 @@ private string GetRestoreArgs(RestoreSettings restoreSettings) args += $" /p:TargetFrameworkRootPath=\"{path}\" /p:NetCoreTargetingPackRoot=\"{path}\" /p:AllowMissingPrunePackageData=true"; } - if (restoreSettings.PathToNugetConfig != null) - { - args += $" --configfile \"{restoreSettings.PathToNugetConfig}\""; - } - if (restoreSettings.ForceReevaluation) { args += " --force"; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs index d14dee506524..394a05e9e596 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs @@ -17,7 +17,7 @@ public interface IDotNet IList GetNugetFeedsFromFolder(string folderPath); } - public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? NugetSources = null, string? PathToNugetConfig = null, bool ForceReevaluation = false, bool TargetWindows = false); + public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? NugetSources = null, bool ForceReevaluation = false, bool TargetWindows = false); public partial record class RestoreResult(bool Success, IList Output) { diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs index a2996497e005..74939b61af5e 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs @@ -130,11 +130,11 @@ public void TestDotnetRestoreProjectToDirectory2() var dotnet = MakeDotnet(dotnetCliInvoker); // Execute - var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, null, "myconfig.config")); + var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, null)); // Verify var lastArgs = dotnetCliInvoker.GetLastArgs(); - Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\"", lastArgs); + Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs); Assert.Equal(2, res.AssetsFilePaths.Count()); Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths); Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths); @@ -148,11 +148,11 @@ public void TestDotnetRestoreProjectToDirectory3() var dotnet = MakeDotnet(dotnetCliInvoker); // Execute - var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, null, "myconfig.config", true)); + var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, null, true)); // Verify var lastArgs = dotnetCliInvoker.GetLastArgs(); - Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\" --force", lastArgs); + Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --force", lastArgs); Assert.Equal(2, res.AssetsFilePaths.Count()); Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths); Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths); From dc1090cd6d3b081cbba468b86e2d2fec5557c2a3 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Mon, 29 Jun 2026 15:42:33 +0200 Subject: [PATCH 06/12] C#: Add some more documentation in the FeedManager. --- .../FeedManager.cs | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index 99568fa9ce19..9694a60d3127 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -26,7 +26,14 @@ internal sealed partial class FeedManager : IDisposable private readonly DependencyDirectory emptyPackageDirectory; private readonly ImmutableHashSet privateRegistryFeeds; + /// + /// Gets whether there are private package registries configured for C#. + /// public bool HasPrivateRegistryFeeds { get; } + + /// + /// Gets whether the reachability of the NuGet feeds should be checked before using them for restore. + /// public bool CheckNugetFeedResponsiveness { get; } = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.CheckNugetFeedResponsiveness); private readonly Lazy> lazyExplicitFeeds; @@ -145,6 +152,13 @@ private IEnumerable GetFeedsFromFolder(string folderPath) => private IEnumerable GetFeedsFromNugetConfig(string nugetConfigPath) => GetFeeds(() => dotnet.GetNugetFeeds(nugetConfigPath)); + /// + /// Constructs the NuGet sources argument for the restore command based on the given feeds. + /// If there are no feeds, a dummy source argument is added to override any default feeds that `restore` would use. + /// + /// The list of feeds to use for the restore command. + /// The prefix to use for each source argument (e.g., "-s"). + /// The constructed NuGet sources argument for the restore command. public string FeedsToRestoreArgument(IEnumerable feeds, string sourceArgumentPrefix) { // If there are no feeds, we want to override any default feeds that `restore` would use by passing a dummy source argument. @@ -194,6 +208,13 @@ public IEnumerable FeedsToUse(string path) return FeedsToUseAux(feedsToConsider); } + /// + /// Constructs the list of NuGet sources to use for this restore. + /// (1) Use the feeds we get from `dotnet nuget list source --configfile` + /// (2) Use private registries, if they are configured + /// + /// Path to the NuGet configuration file. + /// The list of NuGet feeds to use for this restore. public IEnumerable FeedsToUseFromConfig(string config) { var feedsToConsider = GetFeedsFromNugetConfig(config).ToHashSet(); @@ -201,6 +222,11 @@ public IEnumerable FeedsToUseFromConfig(string config) return FeedsToUseAux(feedsToConsider); } + /// + /// Constructs the NuGet sources argument for the `dotnet restore` command based on the given feeds. + /// + /// The list of NuGet feeds to use for the restore command. + /// A string representing the NuGet sources argument for the `dotnet restore` command. public string FeedsToDotnetRestoreArgument(IEnumerable feeds) { return FeedsToRestoreArgument(feeds, "-s"); @@ -212,7 +238,7 @@ public string FeedsToDotnetRestoreArgument(IEnumerable feeds) /// (2) Use private registries, if they are configured /// /// Path to project/solution - /// A string representing the NuGet sources argument for the restore command. + /// A string representing the NuGet sources argument for the `dotnet restore` command. public string? MakeDotnetRestoreSourcesArgument(string path) { // Do not construct a set of explicit NuGet sources to use for restore. @@ -367,6 +393,11 @@ private bool CheckSpecifiedFeeds(ImmutableHashSet feeds, out ImmutableHa return isTimeout; } + /// + /// Return true if the default NuGet feed is reachable, false otherwise. + /// If the reachability check is disabled, this method will always return true. + /// + /// True if the default NuGet feed is reachable, false otherwise. public bool IsDefaultFeedReachable() { if (CheckNugetFeedResponsiveness) From 6ee50cec09a68c62ad413344c27b8079373046f3 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Mon, 29 Jun 2026 16:21:58 +0200 Subject: [PATCH 07/12] C#: Remove the explicitfeeds argument from DownloadMissingPackagesFromSpecificFeeds as it is always called with this argument. --- .../FeedManager.cs | 12 ++++++------ .../NugetPackageRestorer.cs | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index 9694a60d3127..3206c088b27e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -445,7 +445,7 @@ private List GetReachableNuGetFeeds(HashSet feedsToCheck, bool i return reachableFeeds; } - public List GetReachableFallbackNugetFeeds(ImmutableHashSet? feedsFromNugetConfigs) + public List GetReachableFallbackNugetFeeds() { var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet(); if (fallbackFeeds.Count == 0) @@ -456,12 +456,12 @@ public List GetReachableFallbackNugetFeeds(ImmutableHashSet? fee var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback); logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}"); - if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0) + if (shouldAddNugetConfigFeeds && ExplicitFeeds.Count > 0) { - // There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those. - // But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer. - fallbackFeeds.UnionWith(feedsFromNugetConfigs); - logger.LogInfo($"Using NuGet feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}"); + // Feeds in `ExplicitFeeds` may already been checked for reachability. + // But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again. + fallbackFeeds.UnionWith(ExplicitFeeds); + logger.LogInfo($"Using NuGet feeds from nuget.config files as fallback feeds: {string.Join(", ", ExplicitFeeds.OrderBy(f => f))}"); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 8dd04e989edd..c65c2153beaf 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -133,7 +133,7 @@ public HashSet Restore() { // If we experience a timeout, we use this fallback. // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too. - var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds([], explicitFeeds); + var unresponsiveMissingPackageLocation = DownloadMissingPackagesAndUseFallback([]); return unresponsiveMissingPackageLocation is null ? [] : [unresponsiveMissingPackageLocation]; @@ -196,7 +196,7 @@ public HashSet Restore() var usedPackageNames = GetAllUsedPackageDirNames(dependencies); var missingPackageLocation = feedManager.CheckNugetFeedResponsiveness - ? DownloadMissingPackagesFromSpecificFeeds(usedPackageNames, explicitFeeds) + ? DownloadMissingPackagesAndUseFallback(usedPackageNames) : DownloadMissingPackages(usedPackageNames); if (missingPackageLocation is not null) @@ -303,9 +303,9 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag usedPackageNames, ImmutableHashSet? feedsFromNugetConfigs) + private AssemblyLookupLocation? DownloadMissingPackagesAndUseFallback(IEnumerable usedPackageNames) { - var reachableFallbackFeeds = feedManager.GetReachableFallbackNugetFeeds(feedsFromNugetConfigs); + var reachableFallbackFeeds = feedManager.GetReachableFallbackNugetFeeds(); compilationInfoContainer.CompilationInfos.Add(("Reachable fallback NuGet feed count", reachableFallbackFeeds.Count.ToString())); if (reachableFallbackFeeds.Count > 0) From 359408f2880481b272a0428de4a9b8d16815e467 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 30 Jun 2026 15:09:52 +0200 Subject: [PATCH 08/12] C#: Push the feed fetching logic into DownloadPackages. --- .../NugetPackageRestorer.cs | 52 ++++++++----------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index c65c2153beaf..4fd953750044 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -133,7 +133,7 @@ public HashSet Restore() { // If we experience a timeout, we use this fallback. // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too. - var unresponsiveMissingPackageLocation = DownloadMissingPackagesAndUseFallback([]); + var unresponsiveMissingPackageLocation = DownloadMissingPackages([]); return unresponsiveMissingPackageLocation is null ? [] : [unresponsiveMissingPackageLocation]; @@ -195,9 +195,7 @@ public HashSet Restore() var usedPackageNames = GetAllUsedPackageDirNames(dependencies); - var missingPackageLocation = feedManager.CheckNugetFeedResponsiveness - ? DownloadMissingPackagesAndUseFallback(usedPackageNames) - : DownloadMissingPackages(usedPackageNames); + var missingPackageLocation = DownloadMissingPackages(usedPackageNames); if (missingPackageLocation is not null) { @@ -303,22 +301,29 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag usedPackageNames) + private AssemblyLookupLocation? DownloadMissingPackages(IEnumerable usedPackageNames) { - var reachableFallbackFeeds = feedManager.GetReachableFallbackNugetFeeds(); - compilationInfoContainer.CompilationInfos.Add(("Reachable fallback NuGet feed count", reachableFallbackFeeds.Count.ToString())); + IEnumerable? feeds = null; + if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds) + { + // Attempt to get the fallback configuration. + var reachableFallbackFeeds = feedManager.GetReachableFallbackNugetFeeds(); + compilationInfoContainer.CompilationInfos.Add(("Reachable fallback NuGet feed count", reachableFallbackFeeds.Count.ToString())); - if (reachableFallbackFeeds.Count > 0) + if (reachableFallbackFeeds.Count == 0) + { + logger.LogWarning("Skipping download of missing packages from specific feeds as no fallback NuGet feeds are reachable."); + return null; + } + feeds = reachableFallbackFeeds; + } + else if (GetNugetConfig() is string config) { - return DownloadMissingPackages(usedPackageNames, fallbackNugetFeeds: reachableFallbackFeeds); + feeds = feedManager.FeedsToUseFromConfig(config); } - logger.LogWarning("Skipping download of missing packages from specific feeds as no fallback NuGet feeds are reachable."); - return null; - } + var nugetSources = feeds is not null ? feedManager.FeedsToDotnetRestoreArgument(feeds) : null; - private AssemblyLookupLocation? DownloadMissingPackages(IEnumerable usedPackageNames, IEnumerable? fallbackNugetFeeds = null) - { var alreadyDownloadedPackages = usedPackageNames.Select(p => p.ToLowerInvariant()); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); @@ -351,18 +356,6 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag feeds = []; - if (fallbackNugetFeeds is not null) - { - feeds = fallbackNugetFeeds; - } - else if (GetNugetConfig() is string config) - { - feeds = feedManager.FeedsToUseFromConfig(config); - } - - var nugetSources = feedManager.FeedsToDotnetRestoreArgument(feeds); - compilationInfoContainer.CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString())); var successCount = 0; @@ -370,7 +363,7 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag { - var success = TryRestorePackageManually(package.Name, nugetSources, package.PackageReferenceSource, tryWithoutNugetConfig: fallbackNugetFeeds is null); + var success = TryRestorePackageManually(package.Name, nugetSources, package.PackageReferenceSource); if (!success) { return; @@ -476,8 +469,7 @@ private static IEnumerable GetRestoredPackageDirectoryNames(DirectoryInf .Select(d => Path.GetFileName(d).ToLowerInvariant()); } - private bool TryRestorePackageManually(string package, string? nugetSources = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, - bool tryWithoutNugetConfig = true, bool tryPrereleaseVersion = true) + private bool TryRestorePackageManually(string package, string? nugetSources = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryPrereleaseVersion = true) { logger.LogInfo($"Restoring package {package}..."); using var tempDir = new TemporaryDirectory( @@ -505,7 +497,7 @@ private bool TryRestorePackageManually(string package, string? nugetSources = nu return true; } - if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetSources is not null && !feedManager.CheckNugetFeedResponsiveness) + if (!feedManager.CheckNugetFeedResponsiveness && res.HasNugetPackageSourceError && nugetSources is not null) { logger.LogDebug($"Trying to restore '{package}' without nuget.config."); // Restore could not be completed because the listed source is unavailable. Try without the nuget.config: From e934562fcdb0db704afe7ed970548232a10a7f7e Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 30 Jun 2026 15:36:43 +0200 Subject: [PATCH 09/12] C#: Turn reachable fallback feeds into a property. --- .../FeedManager.cs | 12 ++++++++++++ .../NugetPackageRestorer.cs | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index 3206c088b27e..bda94ec95c52 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -78,6 +78,13 @@ internal sealed partial class FeedManager : IDisposable /// public ImmutableHashSet ReachableFeeds => lazyReachableFeeds.Value; + private readonly Lazy> lazyReachableFallbackFeeds; + /// + /// Gets the list of reachable NuGet feeds that are configured as fallback feeds. + /// + public ImmutableHashSet ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value; + + public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotProxy, FileProvider fileProvider) { this.logger = logger; @@ -101,6 +108,11 @@ public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotPr CheckSpecifiedFeeds(InheritedFeeds, out var reachableInheritedFeeds); return ReachableExplicitFeeds.Union(reachableInheritedFeeds).ToImmutableHashSet(); }); + lazyReachableFallbackFeeds = new Lazy>(() => + { + var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(); + return reachableFallbackFeeds.ToImmutableHashSet(); + }); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 4fd953750044..d52710ae3ac5 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -307,7 +307,7 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag Date: Tue, 30 Jun 2026 16:20:38 +0200 Subject: [PATCH 10/12] C#: Ensure that TryRestore provides an explicit nuget sources and make it respect the check nuget feed flag. --- .../NugetPackageRestorer.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index d52710ae3ac5..d053ce15f1ee 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -53,7 +53,9 @@ public NugetPackageRestorer( public string? TryRestore(string package) { - if (TryRestorePackageManually(package)) + var feeds = feedManager.CheckNugetFeedResponsiveness ? feedManager.ReachableFeeds : feedManager.AllFeeds; + var nugetSources = feedManager.FeedsToDotnetRestoreArgument(feeds); + if (TryRestorePackageManually(package, nugetSources)) { var packageDir = DependencyManager.GetPackageDirectory(package, missingPackageDirectory.DirInfo); if (packageDir is not null) @@ -301,7 +303,7 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag usedPackageNames) + private string? GetFallbackNugetSources() { IEnumerable? feeds = null; if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds) @@ -324,6 +326,13 @@ private void RestoreProjects(IEnumerable projects, out ConcurrentBag usedPackageNames) + { + var nugetSources = GetFallbackNugetSources(); + var alreadyDownloadedPackages = usedPackageNames.Select(p => p.ToLowerInvariant()); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); @@ -469,7 +478,7 @@ private static IEnumerable GetRestoredPackageDirectoryNames(DirectoryInf .Select(d => Path.GetFileName(d).ToLowerInvariant()); } - private bool TryRestorePackageManually(string package, string? nugetSources = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryPrereleaseVersion = true) + private bool TryRestorePackageManually(string package, string? nugetSources, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryPrereleaseVersion = true) { logger.LogInfo($"Restoring package {package}..."); using var tempDir = new TemporaryDirectory( From 9aa8a6401835af2dd1bc7135929e3d85907449b7 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 30 Jun 2026 16:39:30 +0200 Subject: [PATCH 11/12] C#: Remove the case where only the fallback is used in case there is a timeout. --- .../NugetPackageRestorer.cs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index d053ce15f1ee..d430e69fa23c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -130,17 +130,6 @@ public HashSet Restore() var allExplicitReachable = explicitFeeds.Count == feedManager.ReachableExplicitFeeds.Count; EmitUnreachableFeedsDiagnostics(allExplicitReachable); - - if (feedManager.ExplicitFeedTimeout) - { - // If we experience a timeout, we use this fallback. - // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too. - var unresponsiveMissingPackageLocation = DownloadMissingPackages([]); - return unresponsiveMissingPackageLocation is null - ? [] - : [unresponsiveMissingPackageLocation]; - } - } try From b8ee7dd20d5053dec1e2f50fd1322ef4d7fe71c2 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Tue, 30 Jun 2026 16:56:04 +0200 Subject: [PATCH 12/12] C#: Remove the timeout specific boolean that is passed around. --- .../FeedManager.cs | 50 ++++++------------- 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index bda94ec95c52..ad5565c67e8a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -60,17 +60,12 @@ internal sealed partial class FeedManager : IDisposable /// public ImmutableHashSet InheritedFeeds => AllFeeds.Except(ExplicitFeeds).ToImmutableHashSet(); - private readonly Lazy<(bool, ImmutableHashSet)> lazyReachableExplicitFeeds; - - /// - /// Gets whether there was a timeout when checking the reachability of the explicitly configured NuGet feeds. - /// - public bool ExplicitFeedTimeout => lazyReachableExplicitFeeds.Value.Item1; + private readonly Lazy> lazyReachableExplicitFeeds; /// /// Gets the list of reachable NuGet feeds that are explicitly configured. /// - public ImmutableHashSet ReachableExplicitFeeds => lazyReachableExplicitFeeds.Value.Item2; + public ImmutableHashSet ReachableExplicitFeeds => lazyReachableExplicitFeeds.Value; private readonly Lazy> lazyReachableFeeds; /// @@ -97,15 +92,11 @@ public FeedManager(ILogger logger, IDotNet dotnet, DependabotProxy? dependabotPr lazyExplicitFeeds = new Lazy>(GetExplicitFeeds); lazyAllFeeds = new Lazy>(GetAllFeeds); - lazyReachableExplicitFeeds = new Lazy<(bool, ImmutableHashSet)>(() => - { - var timeout = CheckSpecifiedFeeds(ExplicitFeeds, out var reachableFeeds); - return (timeout, reachableFeeds); - }); + lazyReachableExplicitFeeds = new Lazy>(() => CheckSpecifiedFeeds(ExplicitFeeds)); lazyReachableFeeds = new Lazy>(() => { // Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific). - CheckSpecifiedFeeds(InheritedFeeds, out var reachableInheritedFeeds); + var reachableInheritedFeeds = CheckSpecifiedFeeds(InheritedFeeds); return ReachableExplicitFeeds.Union(reachableInheritedFeeds).ToImmutableHashSet(); }); lazyReachableFallbackFeeds = new Lazy>(() => @@ -288,7 +279,7 @@ private static async Task ExecuteGetRequest(string address, return await httpClient.GetAsync(address, HttpCompletionOption.ResponseHeadersRead, cancellationToken); } - private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, out bool isTimeout) + private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount) { logger.LogInfo($"Checking if NuGet feed '{feed}' is reachable..."); @@ -321,8 +312,6 @@ private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, using HttpClient client = new(httpClientHandler); - isTimeout = false; - for (var i = 0; i < tryCount; i++) { using var cts = new CancellationTokenSource(); @@ -352,7 +341,6 @@ private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, } logger.LogWarning($"Didn't receive answer from NuGet feed '{feed}'. Tried it {tryCount} times."); - isTimeout = true; return false; } @@ -376,12 +364,10 @@ private HashSet GetExcludedFeeds() /// Checks that we can connect to the specified NuGet feeds. /// /// The set of package feeds to check. - /// The list of feeds that were reachable. /// - /// True if there is a timeout when trying to reach the feeds (excluding any feeds that are configured - /// to be excluded from the check) or false otherwise. + /// The set of feeds that were reachable. /// - private bool CheckSpecifiedFeeds(ImmutableHashSet feeds, out ImmutableHashSet reachableFeeds) + private ImmutableHashSet CheckSpecifiedFeeds(ImmutableHashSet feeds) { // Exclude any feeds from the feed check that are configured by the corresponding environment variable. // These feeds are always assumed to be reachable. @@ -397,12 +383,12 @@ private bool CheckSpecifiedFeeds(ImmutableHashSet feeds, out ImmutableHa return true; }).ToHashSet(); - var reachable = GetReachableNuGetFeeds(feedsToCheck, isFallback: false, out var isTimeout); + var reachable = GetReachableNuGetFeeds(feedsToCheck, isFallback: false); // Always consider feeds excluded for the reachability check as reachable. - reachableFeeds = reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet(); + var reachableFeeds = reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet(); - return isTimeout; + return reachableFeeds; } /// @@ -415,7 +401,7 @@ public bool IsDefaultFeedReachable() if (CheckNugetFeedResponsiveness) { var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false); - return IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount, out var _); + return IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount); } return true; @@ -426,22 +412,15 @@ public bool IsDefaultFeedReachable() /// /// The feeds to check. /// Whether the feeds are fallback feeds or not. - /// Whether a timeout occurred while checking the feeds. /// The list of feeds that could be reached. - private List GetReachableNuGetFeeds(HashSet feedsToCheck, bool isFallback, out bool isTimeout) + private List GetReachableNuGetFeeds(HashSet feedsToCheck, bool isFallback) { var fallbackStr = isFallback ? "fallback " : ""; logger.LogInfo($"Checking {fallbackStr}NuGet feed reachability on feeds: {string.Join(", ", feedsToCheck.OrderBy(f => f))}"); var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback); - var timeout = false; var reachableFeeds = feedsToCheck - .Where(feed => - { - var reachable = IsFeedReachable(feed, initialTimeout, tryCount, out var feedTimeout); - timeout |= feedTimeout; - return reachable; - }) + .Where(feed => IsFeedReachable(feed, initialTimeout, tryCount)) .ToList(); if (reachableFeeds.Count == 0) @@ -453,7 +432,6 @@ private List GetReachableNuGetFeeds(HashSet feedsToCheck, bool i logger.LogInfo($"Reachable {fallbackStr}NuGet feeds: {string.Join(", ", reachableFeeds.OrderBy(f => f))}"); } - isTimeout = timeout; return reachableFeeds; } @@ -477,7 +455,7 @@ public List GetReachableFallbackNugetFeeds() } } - return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true, out var _); + return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true); } private ImmutableHashSet GetExplicitFeeds()