Skip to content

Commit

Permalink
feat(python): add support for poetry dev dependencies
Browse files Browse the repository at this point in the history 
Signed-off-by: nikpivkin <[email protected]>
  • Loading branch information
@nikpivkin
nikpivkin committed Dec 21, 2024
1 parent d1e2450 commit dc14db8
Show file tree
Hide file tree
Showing 5 changed files with 247 additions and 23 deletions.
2 changes: 1 addition & 1 deletion docs/docs/coverage/language/python.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ The following table provides an outline of the features Trivy offers.
|-----------------|------------------|:-----------------------:|:----------------:|:------------------------------------:|:--------:|:----------------------------------------:|
| pip | requirements.txt | - | Include | - | ✓ | ✓ |
| Pipenv | Pipfile.lock | ✓ | Include | - | ✓ | Not needed |
| Poetry | poetry.lock | ✓ | Exclude | ✓ | - | Not needed |
| Poetry | poetry.lock | ✓ | Include | ✓ | - | Not needed |


| Packaging | Dependency graph |
Expand Down
37 changes: 19 additions & 18 deletions pkg/fanal/analyzer/language/python/poetry/poetry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,46 +103,47 @@ func (a poetryAnalyzer) mergePyProject(fsys fs.FS, dir string, app *types.Applic
return xerrors.Errorf("unable to parse %s: %w", path, err)
}

// Identify the direct/transitive dependencies
prodRootDeps := project.Tool.Poetry.Dependencies
directDeps := lo.Assign(prodRootDeps, getDevDeps(project))
prodDeps := getProdPackages(app, prodRootDeps)

// Identify the direct/transitive/dev dependencies
for i, pkg := range app.Packages {
if _, ok := project.Tool.Poetry.Dependencies[pkg.Name]; ok {
_, isProd := prodDeps[pkg.ID]
app.Packages[i].Dev = !isProd
if _, ok := directDeps[pkg.Name]; ok {
app.Packages[i].Relationship = types.RelationshipDirect
} else {
app.Packages[i].Indirect = true
app.Packages[i].Relationship = types.RelationshipIndirect
}
}

filterProdPackages(project, app)
return nil
}

func filterProdPackages(project pyproject.PyProject, app *types.Application) {
func getDevDeps(project pyproject.PyProject) map[string]struct{} {
deps := make(map[string]struct{})
for _, groupDeps := range project.Tool.Poetry.Groups {
deps = lo.Assign(deps, groupDeps.Dependencies)
}
return deps
}

func getProdPackages(app *types.Application, prodRootDeps map[string]struct{}) map[string]struct{} {
packages := lo.SliceToMap(app.Packages, func(pkg types.Package) (string, types.Package) {
return pkg.ID, pkg
})

visited := make(map[string]struct{})
deps := project.Tool.Poetry.Dependencies

for group, groupDeps := range project.Tool.Poetry.Groups {
if group == "dev" {
continue
}
deps = lo.Assign(deps, groupDeps.Dependencies)
}

for _, pkg := range packages {
if _, prodDep := deps[pkg.Name]; !prodDep {
if _, directDep := prodRootDeps[pkg.Name]; !directDep {
continue
}
walkPackageDeps(pkg.ID, packages, visited)
}

app.Packages = lo.Filter(app.Packages, func(pkg types.Package, _ int) bool {
_, ok := visited[pkg.ID]
return ok
})
return visited
}

func walkPackageDeps(pkgID string, packages map[string]types.Package, visited map[string]struct{}) {
Expand Down
125 changes: 123 additions & 2 deletions pkg/fanal/analyzer/language/python/poetry/poetry_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,17 +187,31 @@ func Test_poetryLibraryAnalyzer_Analyze(t *testing.T) {
// export PATH="/root/.local/bin:$PATH"
// poetry new groups && cd groups
// poetry add [email protected]
// poetry add [email protected] --extras socks
// poetry add --optional [email protected]
// poetry add --group dev [email protected]
// poetry add --group lint [email protected]
// poetry add --optional [email protected]
name: "skip deps from groups",
name: "with groups",
dir: "testdata/with-groups",
want: &analyzer.AnalysisResult{
Applications: []types.Application{
{
Type: types.Poetry,
FilePath: "poetry.lock",
Packages: types.Packages{
{
ID: "[email protected]",
Name: "anyio",
Version: "4.7.0",
Indirect: true,
Relationship: types.RelationshipIndirect,
DependsOn: []string{
"[email protected]",
"[email protected]",
"[email protected]",
"[email protected]",
},
},
{
ID: "[email protected]",
Name: "certifi",
Expand All @@ -212,20 +226,105 @@ func Test_poetryLibraryAnalyzer_Analyze(t *testing.T) {
Indirect: true,
Relationship: types.RelationshipIndirect,
},
{
ID: "[email protected]",
Name: "colorama",
Version: "0.4.6",
Indirect: true,
Relationship: types.RelationshipIndirect,
Dev: true,
},
{
ID: "[email protected]",
Name: "exceptiongroup",
Version: "1.2.2",
Indirect: true,
Relationship: types.RelationshipIndirect,
},
{
ID: "[email protected]",
Name: "h11",
Version: "0.14.0",
Indirect: true,
Relationship: types.RelationshipIndirect,
},
{
ID: "[email protected]",
Name: "httpcore",
Version: "1.0.7",
Indirect: true,
Relationship: types.RelationshipIndirect,
DependsOn: []string{
"[email protected]",
"[email protected]",
},
},
{
ID: "[email protected]",
Name: "httpx",
Version: "0.28.1",
Relationship: types.RelationshipDirect,
DependsOn: []string{
"[email protected]",
"[email protected]",
"[email protected]",
"[email protected]",
"[email protected]",
},
},
{
ID: "[email protected]",
Name: "idna",
Version: "3.10",
Indirect: true,
Relationship: types.RelationshipIndirect,
},
{
ID: "[email protected]",
Name: "iniconfig",
Version: "2.0.0",
Indirect: true,
Relationship: types.RelationshipIndirect,
Dev: true,
},
{
ID: "[email protected]",
Name: "mypy-extensions",
Version: "1.0.0",
Indirect: true,
Relationship: types.RelationshipIndirect,
},
{
ID: "[email protected]",
Name: "packaging",
Version: "24.2",
Indirect: true,
Relationship: types.RelationshipIndirect,
Dev: true,
},
{
ID: "[email protected]",
Name: "pluggy",
Version: "1.5.0",
Indirect: true,
Relationship: types.RelationshipIndirect,
Dev: true,
},
{
ID: "[email protected]",
Name: "pytest",
Version: "8.3.4",
Relationship: types.RelationshipDirect,
Dev: true,
DependsOn: []string{
"[email protected]",
"[email protected]",
"[email protected]",
"[email protected]",
"[email protected]",
"[email protected]",
},
},
{
ID: "[email protected]",
Name: "requests",
Expand All @@ -242,8 +341,30 @@ func Test_poetryLibraryAnalyzer_Analyze(t *testing.T) {
ID: "[email protected]",
Name: "ruff",
Version: "0.8.3",
Relationship: types.RelationshipDirect,
Dev: true,
},
{
ID: "[email protected]",
Name: "sniffio",
Version: "1.3.1",
Indirect: true,
Relationship: types.RelationshipIndirect,
},
{
ID: "[email protected]",
Name: "socksio",
Version: "1.0.0",
Indirect: true,
Relationship: types.RelationshipIndirect,
},
{
ID: "[email protected]",
Name: "tomli",
Version: "2.2.1",
Indirect: true,
Relationship: types.RelationshipIndirect,
Dev: true,
},
{
ID: "[email protected]",
Expand Down
105 changes: 103 additions & 2 deletions pkg/fanal/analyzer/language/python/poetry/testdata/with-groups/poetry.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pkg/fanal/analyzer/language/python/poetry/testdata/with-groups/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ readme = "README.md"
python = "^3.9"
requests = "2.32.3"
typing-inspect = {version = "0.9.0", optional = true}
httpx = {version = "0.28.1", extras = ["socks"]}


[tool.poetry.group.dev.dependencies]
Expand Down

0 comments on commit dc14db8

Please sign in to comment.