Skip to content

Commit 82e9c78

Browse files
cataggarcataggar
authored
add uv.lock support (#1425)
* add component type * update UvLockDetectorExperimentTests
1 parent ee30155 commit 82e9c78

File tree

14 files changed

+1549
-0
lines changed

14 files changed

+1549
-0
lines changed

‎docs/detectors/uv.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# uv Detection
2+
## Requirements
3+
[uv](https://docs.astral.sh/uv/) detection relies on a [uv.lock](https://docs.astral.sh/uv/concepts/projects/layout/#the-lockfile) file being present.
4+
5+
## Detection strategy
6+
uv detection is performed by parsing a <em>uv.lock</em> found under the scan directory.

‎src/Microsoft.ComponentDetection.Detectors/uv/UvDependency.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#nullable enable
2+
namespace Microsoft.ComponentDetection.Detectors.Uv
3+
{
4+
public class UvDependency
5+
{
6+
public required string Name { get; init; }
7+
8+
public string? Specifier { get; set; }
9+
}
10+
}

‎src/Microsoft.ComponentDetection.Detectors/uv/UvLock.cs‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
#nullable enable
2+
namespace Microsoft.ComponentDetection.Detectors.Uv
3+
{
4+
using System;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using Tomlyn;
8+
using Tomlyn.Model;
9+
10+
public class UvLock
11+
{
12+
// a list of packages with their dependencies
13+
public List<UvPackage> Packages { get; set; } = [];
14+
15+
// static method to parse the TOML stream into a UvLock model
16+
public static UvLock Parse(Stream tomlStream)
17+
{
18+
using var reader = new StreamReader(tomlStream);
19+
var tomlContent = reader.ReadToEnd();
20+
var model = Toml.ToModel(tomlContent);
21+
return new UvLock
22+
{
23+
Packages = ParsePackagesFromModel(model),
24+
};
25+
}
26+
27+
internal static List<UvPackage> ParsePackagesFromModel(object? model)
28+
{
29+
if (model is not TomlTable table)
30+
{
31+
throw new InvalidOperationException("TOML root is not a table");
32+
}
33+
34+
if (!table.TryGetValue("package", out var packagesObj) || packagesObj is not TomlTableArray packages)
35+
{
36+
return [];
37+
}
38+
39+
var result = new List<UvPackage>();
40+
foreach (var pkg in packages)
41+
{
42+
var parsed = ParsePackage(pkg);
43+
if (parsed is not null)
44+
{
45+
result.Add(parsed);
46+
}
47+
}
48+
49+
return result;
50+
}
51+
52+
internal static UvPackage? ParsePackage(object? pkg)
53+
{
54+
if (pkg is not TomlTable pkgTable)
55+
{
56+
return null;
57+
}
58+
59+
if (pkgTable.TryGetValue("name", out var nameObj) && nameObj is string name &&
60+
pkgTable.TryGetValue("version", out var versionObj) && versionObj is string version)
61+
{
62+
var uvPackage = new UvPackage
63+
{
64+
Name = name,
65+
Version = version,
66+
Dependencies = [],
67+
MetadataRequiresDist = [],
68+
MetadataRequiresDev = [],
69+
};
70+
71+
if (pkgTable.TryGetValue("dependencies", out var depsObj) && depsObj is TomlArray depsArray)
72+
{
73+
uvPackage.Dependencies = ParseDependenciesArray(depsArray);
74+
}
75+
76+
if (pkgTable.TryGetValue("metadata", out var metadataObj) && metadataObj is TomlTable metadataTable)
77+
{
78+
ParseMetadata(metadataTable, uvPackage);
79+
}
80+
81+
// Parse source
82+
if (pkgTable.TryGetValue("source", out var sourceObj) && sourceObj is TomlTable sourceTable)
83+
{
84+
var source = new UvSource
85+
{
86+
Registry = sourceTable.TryGetValue("registry", out var regObj) && regObj is string reg ? reg : null,
87+
Virtual = sourceTable.TryGetValue("virtual", out var virtObj) && virtObj is string virt ? virt : null,
88+
};
89+
uvPackage.Source = source;
90+
}
91+
92+
return uvPackage;
93+
}
94+
95+
return null;
96+
}
97+
98+
internal static List<UvDependency> ParseDependenciesArray(TomlArray? depsArray)
99+
{
100+
var deps = new List<UvDependency>();
101+
if (depsArray is null)
102+
{
103+
return deps;
104+
}
105+
106+
foreach (var dep in depsArray)
107+
{
108+
if (dep is TomlTable depTable &&
109+
depTable.TryGetValue("name", out var depNameObj) && depNameObj is string depName)
110+
{
111+
var depSpec = depTable.TryGetValue("specifier", out var specObj) && specObj is string s ? s : null;
112+
deps.Add(new UvDependency
113+
{
114+
Name = depName,
115+
Specifier = depSpec,
116+
});
117+
}
118+
}
119+
120+
return deps;
121+
}
122+
123+
internal static void ParseMetadata(TomlTable? metadataTable, UvPackage uvPackage)
124+
{
125+
if (metadataTable is null)
126+
{
127+
return;
128+
}
129+
130+
if (metadataTable.TryGetValue("requires-dist", out var requiresDistObj) && requiresDistObj is TomlArray requiresDistArr)
131+
{
132+
uvPackage.MetadataRequiresDist = ParseDependenciesArray(requiresDistArr);
133+
}
134+
135+
if (metadataTable.TryGetValue("requires-dev", out var requiresDevObj) && requiresDevObj is TomlTable requiresDevTable)
136+
{
137+
if (requiresDevTable.TryGetValue("dev", out var devObj) && devObj is TomlArray devArr)
138+
{
139+
uvPackage.MetadataRequiresDev = ParseDependenciesArray(devArr);
140+
}
141+
}
142+
}
143+
}
144+
}

‎src/Microsoft.ComponentDetection.Detectors/uv/UvLockComponentDetector.cs‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
#nullable enable
2+
3+
namespace Microsoft.ComponentDetection.Detectors.Uv
4+
{
5+
using System;
6+
using System.Collections.Generic;
7+
using System.Linq;
8+
using System.Threading;
9+
using System.Threading.Tasks;
10+
using Microsoft.ComponentDetection.Contracts;
11+
using Microsoft.ComponentDetection.Contracts.Internal;
12+
using Microsoft.ComponentDetection.Contracts.TypedComponent;
13+
using Microsoft.Extensions.Logging;
14+
15+
public class UvLockComponentDetector : FileComponentDetector, IDefaultOffComponentDetector
16+
{
17+
public UvLockComponentDetector(
18+
IComponentStreamEnumerableFactory componentStreamEnumerableFactory,
19+
IObservableDirectoryWalkerFactory walkerFactory,
20+
ILogger<UvLockComponentDetector> logger)
21+
{
22+
this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory;
23+
this.Scanner = walkerFactory;
24+
this.Logger = logger;
25+
}
26+
27+
public override string Id => "UvLock";
28+
29+
public override IList<string> SearchPatterns { get; } = ["uv.lock"];
30+
31+
public override IEnumerable<ComponentType> SupportedComponentTypes => [ComponentType.Pip];
32+
33+
public override int Version => 1;
34+
35+
public override IEnumerable<string> Categories => ["Python"];
36+
37+
internal static bool IsRootPackage(UvPackage pck)
38+
{
39+
return pck.Source?.Virtual != null;
40+
}
41+
42+
protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default)
43+
{
44+
var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder;
45+
var file = processRequest.ComponentStream;
46+
47+
try
48+
{
49+
// Parse the file stream into a UvLock model
50+
file.Stream.Position = 0; // Ensure stream is at the beginning
51+
var uvLock = UvLock.Parse(file.Stream);
52+
53+
var rootPackage = uvLock.Packages.FirstOrDefault(IsRootPackage);
54+
var explicitPackages = new HashSet<string>();
55+
var devPackages = new HashSet<string>();
56+
57+
if (rootPackage != null)
58+
{
59+
foreach (var dep in rootPackage.MetadataRequiresDist)
60+
{
61+
explicitPackages.Add(dep.Name);
62+
}
63+
64+
foreach (var devDep in rootPackage.MetadataRequiresDev)
65+
{
66+
devPackages.Add(devDep.Name);
67+
}
68+
}
69+
70+
foreach (var pkg in uvLock.Packages)
71+
{
72+
if (IsRootPackage(pkg))
73+
{
74+
continue;
75+
}
76+
77+
var pipComponent = new PipComponent(pkg.Name, pkg.Version);
78+
var isExplicit = explicitPackages.Contains(pkg.Name);
79+
var isDev = devPackages.Contains(pkg.Name);
80+
var detectedComponent = new DetectedComponent(pipComponent);
81+
singleFileComponentRecorder.RegisterUsage(detectedComponent, isDevelopmentDependency: isDev, isExplicitReferencedDependency: isExplicit);
82+
83+
foreach (var dep in pkg.Dependencies)
84+
{
85+
var depPkg = uvLock.Packages.FirstOrDefault(p => p.Name.Equals(dep.Name, StringComparison.OrdinalIgnoreCase));
86+
if (depPkg != null)
87+
{
88+
var depComponentWithVersion = new PipComponent(depPkg.Name, depPkg.Version);
89+
singleFileComponentRecorder.RegisterUsage(new DetectedComponent(depComponentWithVersion), parentComponentId: pipComponent.Id);
90+
}
91+
else
92+
{
93+
this.Logger.LogWarning("Dependency {DependencyName} not found in uv.lock packages", dep.Name);
94+
}
95+
}
96+
}
97+
}
98+
catch (Exception ex)
99+
{
100+
this.Logger.LogError(ex, "Failed to parse uv.lock file {File}", file.Location);
101+
}
102+
103+
return Task.CompletedTask;
104+
}
105+
}
106+
}

‎src/Microsoft.ComponentDetection.Detectors/uv/UvPackage.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#nullable enable
2+
namespace Microsoft.ComponentDetection.Detectors.Uv
3+
{
4+
using System.Collections.Generic;
5+
6+
public class UvPackage
7+
{
8+
public required string Name { get; init; }
9+
10+
public required string Version { get; init; }
11+
12+
public List<UvDependency> Dependencies { get; set; } = [];
13+
14+
// Metadata dependencies (requires-dist)
15+
public List<UvDependency> MetadataRequiresDist { get; set; } = [];
16+
17+
// Metadata dev dependencies (requires-dev)
18+
public List<UvDependency> MetadataRequiresDev { get; set; } = [];
19+
20+
// Source property for uv.lock
21+
public UvSource? Source { get; set; }
22+
}
23+
}

‎src/Microsoft.ComponentDetection.Detectors/uv/UvSource.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#nullable enable
2+
3+
namespace Microsoft.ComponentDetection.Detectors.Uv
4+
{
5+
public class UvSource
6+
{
7+
public string? Registry { get; set; }
8+
9+
public string? Virtual { get; set; }
10+
}
11+
}

‎src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/UvLockDetectorExperiment.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
2+
3+
using Microsoft.ComponentDetection.Contracts;
4+
using Microsoft.ComponentDetection.Detectors.Pip;
5+
using Microsoft.ComponentDetection.Detectors.Uv;
6+
7+
/// <summary>
8+
/// Experiment to validate UvLockComponentDetector against PipComponentDetector.
9+
/// </summary>
10+
public class UvLockDetectorExperiment : IExperimentConfiguration
11+
{
12+
/// <inheritdoc />
13+
public string Name => "UvLockDetectorExperiment";
14+
15+
/// <inheritdoc />
16+
public bool IsInControlGroup(IComponentDetector componentDetector) => componentDetector is PipReportComponentDetector;
17+
18+
/// <inheritdoc />
19+
public bool IsInExperimentGroup(IComponentDetector componentDetector) => componentDetector is UvLockComponentDetector;
20+
21+
/// <inheritdoc />
22+
public bool ShouldRecord(IComponentDetector componentDetector, int numComponents) => true;
23+
}

‎src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ namespace Microsoft.ComponentDetection.Orchestrator.Extensions;
2121
using Microsoft.ComponentDetection.Detectors.Rust;
2222
using Microsoft.ComponentDetection.Detectors.Spdx;
2323
using Microsoft.ComponentDetection.Detectors.Swift;
24+
using Microsoft.ComponentDetection.Detectors.Uv;
2425
using Microsoft.ComponentDetection.Detectors.Vcpkg;
2526
using Microsoft.ComponentDetection.Detectors.Yarn;
2627
using Microsoft.ComponentDetection.Detectors.Yarn.Parsers;
@@ -67,6 +68,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s
6768
services.AddSingleton<IExperimentConfiguration, RustCliDetectorExperiment>();
6869
services.AddSingleton<IExperimentConfiguration, RustSbomVsCliExperiment>();
6970
services.AddSingleton<IExperimentConfiguration, RustSbomVsCrateExperiment>();
71+
services.AddSingleton<IExperimentConfiguration, UvLockDetectorExperiment>();
7072

7173
// Detectors
7274
// CocoaPods
@@ -152,6 +154,9 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s
152154
// Swift Package Manager
153155
services.AddSingleton<IComponentDetector, SwiftResolvedComponentDetector>();
154156

157+
// uv
158+
services.AddSingleton<IComponentDetector, UvLockComponentDetector>();
159+
155160
return services;
156161
}
157162
}

‎test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
<PackageReference Include="System.Reactive" />
1515
<PackageReference Include="System.Threading.Tasks.Dataflow" />
1616
<PackageReference Include="packageurl-dotnet" />
17+
<PackageReference Include="Tomlyn.Signed" />
1718
<PackageReference Include="YamlDotNet" />
1819
<PackageReference Include="MSTest.TestAdapter" />
1920
<PackageReference Include="Microsoft.NET.Test.Sdk" />

0 commit comments

Comments
 (0)