Skip to content

Commit

Permalink
feat(python): add support for uv dev and optional dependencies (#8134)
Browse files Browse the repository at this point in the history 
Signed-off-by: nikpivkin <[email protected]>
  • Loading branch information
@nikpivkin
nikpivkin authored Dec 24, 2024
1 parent 774e04d commit 49c54b4
Show file tree
Hide file tree
Showing 6 changed files with 421 additions and 68 deletions.
4 changes: 3 additions & 1 deletion docs/docs/coverage/language/python.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ The following table provides an outline of the features Trivy offers.
| pip | requirements.txt | - | Include | - | ✓ | ✓ |
| Pipenv | Pipfile.lock | ✓ | Include | - | ✓ | Not needed |
| Poetry | poetry.lock | ✓ | [Exclude](#poetry) | ✓ | - | Not needed |
| uv | uv.lock | ✓ | Exclude | ✓ | - | Not needed |
| uv | uv.lock | ✓ | [Exclude](#uv) | ✓ | - | Not needed | |


| Packaging | Dependency graph |
Expand Down Expand Up @@ -136,6 +136,8 @@ Trivy uses `uv.lock` to identify dependencies and find vulnerabilities.

License detection is not supported for `uv`.

By default, Trivy doesn't report development dependencies. Use the `--include-dev-deps` flag to include them.

## Packaging
Trivy parses the manifest files of installed packages in container image scanning and so on.
See [here](https://packaging.python.org/en/latest/discussions/package-formats/) for the detail.
Expand Down
64 changes: 42 additions & 22 deletions pkg/dependency/parser/python/uv/parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,6 @@ func (l Lock) packages() map[string]Package {
})
}

func (l Lock) directDeps(root Package) set.Set[string] {
deps := set.New[string]()
for _, dep := range root.Dependencies {
deps.Append(dep.Name)
}
return deps
}

func prodDeps(root Package, packages map[string]Package) set.Set[string] {
visited := set.New[string]()
walkPackageDeps(root, packages, visited)
Expand All @@ -42,8 +34,8 @@ func walkPackageDeps(pkg Package, packages map[string]Package, visited set.Set[s
return
}
visited.Append(pkg.Name)
for _, dep := range pkg.Dependencies {
depPkg, exists := packages[dep.Name]
for depName := range pkg.nonDevDeps().Iter() {
depPkg, exists := packages[depName]
if !exists {
continue
}
Expand All @@ -69,10 +61,41 @@ func (l Lock) root() (Package, error) {
}

type Package struct {
Name string `toml:"name"`
Version string `toml:"version"`
Source Source `toml:"source"`
Dependencies []Dependency `toml:"dependencies"`
Name string `toml:"name"`
Version string `toml:"version"`
Source Source `toml:"source"`
Dependencies Dependencies `toml:"dependencies"`
DevDependencies map[string]Dependencies `toml:"dev-dependencies"`
OptionalDependencies map[string]Dependencies `toml:"optional-dependencies"`
}

func (p Package) directDeps() set.Set[string] {
deps := p.nonDevDeps()
for _, groupDeps := range p.DevDependencies {
deps.Append(groupDeps.toSet().Items()...)

}
return deps
}

func (p Package) nonDevDeps() set.Set[string] {
deps := p.Dependencies.toSet()
for _, groupDeps := range p.OptionalDependencies {
deps.Append(groupDeps.toSet().Items()...)
}
return deps
}

type Dependencies []struct {
Name string `toml:"name"`
}

func (d Dependencies) toSet() set.Set[string] {
deps := set.New[string]()
for _, dep := range d {
deps.Append(dep.Name)
}
return deps
}

// https://github.com/astral-sh/uv/blob/f7d647e81d7e1e3be189324b06024ed2057168e6/crates/uv-resolver/src/lock/mod.rs#L572-L579
Expand Down Expand Up @@ -107,7 +130,7 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc
}

packages := lock.packages()
directDeps := lock.directDeps(rootPackage)
directDeps := rootPackage.directDeps()

// Since each lockfile contains a root package with a list of direct dependencies,
// we can identify all production dependencies by traversing the dependency graph
Expand All @@ -120,10 +143,6 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc
)

for _, pkg := range lock.Packages {
if !prodDeps.Contains(pkg.Name) {
continue
}

pkgID := packageID(pkg.Name, pkg.Version)
relationship := ftypes.RelationshipIndirect
if pkg.isRoot() {
Expand All @@ -137,16 +156,17 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc
Name: pkg.Name,
Version: pkg.Version,
Relationship: relationship,
Dev: !prodDeps.Contains(pkg.Name),
})

dependsOn := make([]string, 0, len(pkg.Dependencies))

for _, dep := range pkg.Dependencies {
depPkg, exists := packages[dep.Name]
for depName := range pkg.directDeps().Iter() {
depPkg, exists := packages[depName]
if !exists {
continue
}
dependsOn = append(dependsOn, packageID(dep.Name, depPkg.Version))
dependsOn = append(dependsOn, packageID(depName, depPkg.Version))
}

if len(dependsOn) > 0 {
Expand Down
23 changes: 22 additions & 1 deletion pkg/dependency/parser/python/uv/parse_testcase.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,44 @@ var (
// uv init normal && cd normal
// uv add requests==2.32.0
// uv add --group dev pytest==8.3.4
// uv add httpx==0.28.1 --extra socks
// uv add orjson==3.10.12 --optional json
// apk add jq
// uv pip list --format json |jq -c 'sort_by(.name) | .[] | {"ID": (.name + "@" + .version), "Name": .name, "Version": .version}' | sed 's/$/,/' | sed 's/\"\([^"]*\)\":/\1:/g'

// add a root project
// fill in the relationships between the packages
uvNormal = []ftypes.Package{
{ID: "[email protected]", Name: "normal", Version: "0.1.0", Relationship: ftypes.RelationshipRoot},
{ID: "[email protected]", Name: "httpx", Version: "0.28.1", Relationship: ftypes.RelationshipDirect},
{ID: "[email protected]", Name: "orjson", Version: "3.10.12", Relationship: ftypes.RelationshipDirect},
{ID: "[email protected]", Name: "pytest", Version: "8.3.4", Relationship: ftypes.RelationshipDirect, Dev: true},
{ID: "[email protected]", Name: "requests", Version: "2.32.0", Relationship: ftypes.RelationshipDirect},
{ID: "[email protected]", Name: "anyio", Version: "4.7.0", Relationship: ftypes.RelationshipIndirect},
{ID: "[email protected]", Name: "certifi", Version: "2024.12.14", Relationship: ftypes.RelationshipIndirect},
{ID: "[email protected]", Name: "charset-normalizer", Version: "3.4.0", Relationship: ftypes.RelationshipIndirect},
{ID: "[email protected]", Name: "colorama", Version: "0.4.6", Relationship: ftypes.RelationshipIndirect, Dev: true},
{ID: "[email protected]", Name: "exceptiongroup", Version: "1.2.2", Relationship: ftypes.RelationshipIndirect},
{ID: "[email protected]", Name: "h11", Version: "0.14.0", Relationship: ftypes.RelationshipIndirect},
{ID: "[email protected]", Name: "httpcore", Version: "1.0.7", Relationship: ftypes.RelationshipIndirect},
{ID: "[email protected]", Name: "idna", Version: "3.10", Relationship: ftypes.RelationshipIndirect},
{ID: "[email protected]", Name: "iniconfig", Version: "2.0.0", Relationship: ftypes.RelationshipIndirect, Dev: true},
{ID: "[email protected]", Name: "packaging", Version: "24.2", Relationship: ftypes.RelationshipIndirect, Dev: true},
{ID: "[email protected]", Name: "pluggy", Version: "1.5.0", Relationship: ftypes.RelationshipIndirect, Dev: true},
{ID: "[email protected]", Name: "sniffio", Version: "1.3.1", Relationship: ftypes.RelationshipIndirect},
{ID: "[email protected]", Name: "socksio", Version: "1.0.0", Relationship: ftypes.RelationshipIndirect},
{ID: "[email protected]", Name: "tomli", Version: "2.2.1", Relationship: ftypes.RelationshipIndirect, Dev: true},
{ID: "[email protected]", Name: "typing-extensions", Version: "4.12.2", Relationship: ftypes.RelationshipIndirect},
{ID: "[email protected]", Name: "urllib3", Version: "2.2.3", Relationship: ftypes.RelationshipIndirect},
}

// add a root project
uvNormalDeps = []ftypes.Dependency{
{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]"}},
{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
}
)
Loading

0 comments on commit 49c54b4

Please sign in to comment.