From 4ab4d2b79ac695bd992b80ee99dbd96553c63510 Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Sat, 21 Dec 2024 16:38:16 +0600 Subject: [PATCH 1/5] feat(python): add support for poetry dev dependencies Signed-off-by: nikpivkin --- docs/docs/coverage/language/python.md | 2 +- .../analyzer/language/python/poetry/poetry.go | 37 +++--- .../language/python/poetry/poetry_test.go | 125 +++++++++++++++++- .../poetry/testdata/with-groups/poetry.lock | 105 ++++++++++++++- .../testdata/with-groups/pyproject.toml | 1 + 5 files changed, 247 insertions(+), 23 deletions(-) diff --git a/docs/docs/coverage/language/python.md b/docs/docs/coverage/language/python.md index 9396b04b83eb..00c9bf585a3d 100644 --- a/docs/docs/coverage/language/python.md +++ b/docs/docs/coverage/language/python.md @@ -26,7 +26,7 @@ The following table provides an outline of the features Trivy offers. |-----------------|------------------|:-----------------------:|:----------------:|:------------------------------------:|:--------:|:----------------------------------------:| | pip | requirements.txt | - | Include | - | ✓ | ✓ | | Pipenv | Pipfile.lock | ✓ | Include | - | ✓ | Not needed | -| Poetry | poetry.lock | ✓ | Exclude | ✓ | - | Not needed | +| Poetry | poetry.lock | ✓ | Include | ✓ | - | Not needed | | uv | uv.lock | ✓ | Exclude | ✓ | - | Not needed | diff --git a/pkg/fanal/analyzer/language/python/poetry/poetry.go b/pkg/fanal/analyzer/language/python/poetry/poetry.go index 84073459ea0b..d8c6e15b1c16 100644 --- a/pkg/fanal/analyzer/language/python/poetry/poetry.go +++ b/pkg/fanal/analyzer/language/python/poetry/poetry.go @@ -103,46 +103,47 @@ func (a poetryAnalyzer) mergePyProject(fsys fs.FS, dir string, app *types.Applic return xerrors.Errorf("unable to parse %s: %w", path, err) } - // Identify the direct/transitive dependencies + prodRootDeps := project.Tool.Poetry.Dependencies + directDeps := lo.Assign(prodRootDeps, getDevDeps(project)) + prodDeps := getProdPackages(app, prodRootDeps) + + // Identify the direct/transitive/dev dependencies for i, pkg := range app.Packages { - if _, ok := project.Tool.Poetry.Dependencies[pkg.Name]; ok { + _, isProd := prodDeps[pkg.ID] + app.Packages[i].Dev = !isProd + if _, ok := directDeps[pkg.Name]; ok { app.Packages[i].Relationship = types.RelationshipDirect } else { app.Packages[i].Indirect = true app.Packages[i].Relationship = types.RelationshipIndirect } } - - filterProdPackages(project, app) return nil } -func filterProdPackages(project pyproject.PyProject, app *types.Application) { +func getDevDeps(project pyproject.PyProject) map[string]struct{} { + deps := make(map[string]struct{}) + for _, groupDeps := range project.Tool.Poetry.Groups { + deps = lo.Assign(deps, groupDeps.Dependencies) + } + return deps +} + +func getProdPackages(app *types.Application, prodRootDeps map[string]struct{}) map[string]struct{} { packages := lo.SliceToMap(app.Packages, func(pkg types.Package) (string, types.Package) { return pkg.ID, pkg }) visited := make(map[string]struct{}) - deps := project.Tool.Poetry.Dependencies - - for group, groupDeps := range project.Tool.Poetry.Groups { - if group == "dev" { - continue - } - deps = lo.Assign(deps, groupDeps.Dependencies) - } for _, pkg := range packages { - if _, prodDep := deps[pkg.Name]; !prodDep { + if _, directDep := prodRootDeps[pkg.Name]; !directDep { continue } walkPackageDeps(pkg.ID, packages, visited) } - app.Packages = lo.Filter(app.Packages, func(pkg types.Package, _ int) bool { - _, ok := visited[pkg.ID] - return ok - }) + return visited } func walkPackageDeps(pkgID string, packages map[string]types.Package, visited map[string]struct{}) { diff --git a/pkg/fanal/analyzer/language/python/poetry/poetry_test.go b/pkg/fanal/analyzer/language/python/poetry/poetry_test.go index 2760be6c76df..618b9c8c5d9c 100644 --- a/pkg/fanal/analyzer/language/python/poetry/poetry_test.go +++ b/pkg/fanal/analyzer/language/python/poetry/poetry_test.go @@ -187,10 +187,11 @@ func Test_poetryLibraryAnalyzer_Analyze(t *testing.T) { // export PATH="/root/.local/bin:$PATH" // poetry new groups && cd groups // poetry add requests@2.32.3 + // poetry add httpx@0.28.1 --extras socks + // poetry add --optional typing-inspect@0.9.0 // poetry add --group dev pytest@8.3.4 // poetry add --group lint ruff@0.8.3 - // poetry add --optional typing-inspect@0.9.0 - name: "skip deps from groups", + name: "with groups", dir: "testdata/with-groups", want: &analyzer.AnalysisResult{ Applications: []types.Application{ @@ -198,6 +199,19 @@ func Test_poetryLibraryAnalyzer_Analyze(t *testing.T) { Type: types.Poetry, FilePath: "poetry.lock", Packages: types.Packages{ + { + ID: "anyio@4.7.0", + Name: "anyio", + Version: "4.7.0", + Indirect: true, + Relationship: types.RelationshipIndirect, + DependsOn: []string{ + "exceptiongroup@1.2.2", + "idna@3.10", + "sniffio@1.3.1", + "typing-extensions@4.12.2", + }, + }, { ID: "certifi@2024.12.14", Name: "certifi", @@ -212,6 +226,52 @@ func Test_poetryLibraryAnalyzer_Analyze(t *testing.T) { Indirect: true, Relationship: types.RelationshipIndirect, }, + { + ID: "colorama@0.4.6", + Name: "colorama", + Version: "0.4.6", + Indirect: true, + Relationship: types.RelationshipIndirect, + Dev: true, + }, + { + ID: "exceptiongroup@1.2.2", + Name: "exceptiongroup", + Version: "1.2.2", + Indirect: true, + Relationship: types.RelationshipIndirect, + }, + { + ID: "h11@0.14.0", + Name: "h11", + Version: "0.14.0", + Indirect: true, + Relationship: types.RelationshipIndirect, + }, + { + ID: "httpcore@1.0.7", + Name: "httpcore", + Version: "1.0.7", + Indirect: true, + Relationship: types.RelationshipIndirect, + DependsOn: []string{ + "certifi@2024.12.14", + "h11@0.14.0", + }, + }, + { + ID: "httpx@0.28.1", + Name: "httpx", + Version: "0.28.1", + Relationship: types.RelationshipDirect, + DependsOn: []string{ + "anyio@4.7.0", + "certifi@2024.12.14", + "httpcore@1.0.7", + "idna@3.10", + "socksio@1.0.0", + }, + }, { ID: "idna@3.10", Name: "idna", @@ -219,6 +279,14 @@ func Test_poetryLibraryAnalyzer_Analyze(t *testing.T) { Indirect: true, Relationship: types.RelationshipIndirect, }, + { + ID: "iniconfig@2.0.0", + Name: "iniconfig", + Version: "2.0.0", + Indirect: true, + Relationship: types.RelationshipIndirect, + Dev: true, + }, { ID: "mypy-extensions@1.0.0", Name: "mypy-extensions", @@ -226,6 +294,37 @@ func Test_poetryLibraryAnalyzer_Analyze(t *testing.T) { Indirect: true, Relationship: types.RelationshipIndirect, }, + { + ID: "packaging@24.2", + Name: "packaging", + Version: "24.2", + Indirect: true, + Relationship: types.RelationshipIndirect, + Dev: true, + }, + { + ID: "pluggy@1.5.0", + Name: "pluggy", + Version: "1.5.0", + Indirect: true, + Relationship: types.RelationshipIndirect, + Dev: true, + }, + { + ID: "pytest@8.3.4", + Name: "pytest", + Version: "8.3.4", + Relationship: types.RelationshipDirect, + Dev: true, + DependsOn: []string{ + "colorama@0.4.6", + "exceptiongroup@1.2.2", + "iniconfig@2.0.0", + "packaging@24.2", + "pluggy@1.5.0", + "tomli@2.2.1", + }, + }, { ID: "requests@2.32.3", Name: "requests", @@ -242,8 +341,30 @@ func Test_poetryLibraryAnalyzer_Analyze(t *testing.T) { ID: "ruff@0.8.3", Name: "ruff", Version: "0.8.3", + Relationship: types.RelationshipDirect, + Dev: true, + }, + { + ID: "sniffio@1.3.1", + Name: "sniffio", + Version: "1.3.1", + Indirect: true, + Relationship: types.RelationshipIndirect, + }, + { + ID: "socksio@1.0.0", + Name: "socksio", + Version: "1.0.0", + Indirect: true, + Relationship: types.RelationshipIndirect, + }, + { + ID: "tomli@2.2.1", + Name: "tomli", + Version: "2.2.1", Indirect: true, Relationship: types.RelationshipIndirect, + Dev: true, }, { ID: "typing-extensions@4.12.2", diff --git a/pkg/fanal/analyzer/language/python/poetry/testdata/with-groups/poetry.lock b/pkg/fanal/analyzer/language/python/poetry/testdata/with-groups/poetry.lock index 8172db642ab6..f90fb4513a70 100644 --- a/pkg/fanal/analyzer/language/python/poetry/testdata/with-groups/poetry.lock +++ b/pkg/fanal/analyzer/language/python/poetry/testdata/with-groups/poetry.lock @@ -1,5 +1,27 @@ # This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. +[[package]] +name = "anyio" +version = "4.7.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.9" +files = [ + {file = "anyio-4.7.0-py3-none-any.whl", hash = "sha256:ea60c3723ab42ba6fff7e8ccb0488c898ec538ff4df1f1d5e642c3601d07e352"}, + {file = "anyio-4.7.0.tar.gz", hash = "sha256:2f834749c602966b7d456a7567cafcb309f96482b5081d14ac93ccd457f9dd48"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} + +[package.extras] +doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] +trio = ["trio (>=0.26.1)"] + [[package]] name = "certifi" version = "2024.12.14" @@ -150,6 +172,63 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "1.0.7" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"}, + {file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] + +[[package]] +name = "httpx" +version = "0.28.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, + {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" +socksio = {version = "==1.*", optional = true, markers = "extra == \"socks\""} + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "idna" version = "3.10" @@ -282,6 +361,28 @@ files = [ {file = "ruff-0.8.3.tar.gz", hash = "sha256:5e7558304353b84279042fc584a4f4cb8a07ae79b2bf3da1a7551d960b5626d3"}, ] +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + +[[package]] +name = "socksio" +version = "1.0.0" +description = "Sans-I/O implementation of SOCKS4, SOCKS4A, and SOCKS5." +optional = false +python-versions = ">=3.6" +files = [ + {file = "socksio-1.0.0-py3-none-any.whl", hash = "sha256:95dc1f15f9b34e8d7b16f06d74b8ccf48f609af32ab33c608d08761c5dcbb1f3"}, + {file = "socksio-1.0.0.tar.gz", hash = "sha256:f88beb3da5b5c38b9890469de67d0cb0f9d494b78b106ca1845f96c10b91c4ac"}, +] + [[package]] name = "tomli" version = "2.2.1" @@ -327,7 +428,7 @@ files = [ name = "typing-extensions" version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" -optional = true +optional = false python-versions = ">=3.8" files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, @@ -369,4 +470,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "fd0e5fc1dfc09e39acec1d9fe304fecccbdff1ca85271ab1f15a19d008db23cf" \ No newline at end of file +content-hash = "eb9992c7bdd3f378ae8559495b790f625f0368d978b5664820fa06b2db854bff" \ No newline at end of file diff --git a/pkg/fanal/analyzer/language/python/poetry/testdata/with-groups/pyproject.toml b/pkg/fanal/analyzer/language/python/poetry/testdata/with-groups/pyproject.toml index df7c1e2718a7..919afb32bb68 100644 --- a/pkg/fanal/analyzer/language/python/poetry/testdata/with-groups/pyproject.toml +++ b/pkg/fanal/analyzer/language/python/poetry/testdata/with-groups/pyproject.toml @@ -9,6 +9,7 @@ readme = "README.md" python = "^3.9" requests = "2.32.3" typing-inspect = {version = "0.9.0", optional = true} +httpx = {version = "0.28.1", extras = ["socks"]} [tool.poetry.group.dev.dependencies] From 6186b7dae0c6b9f7daa5bd2357e515c5a56a2e09 Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Mon, 23 Dec 2024 23:35:38 +0600 Subject: [PATCH 2/5] docs: add note about --include-dev-deps Signed-off-by: nikpivkin --- docs/docs/coverage/language/python.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/docs/coverage/language/python.md b/docs/docs/coverage/language/python.md index 00c9bf585a3d..3033b2d71941 100644 --- a/docs/docs/coverage/language/python.md +++ b/docs/docs/coverage/language/python.md @@ -128,6 +128,9 @@ To build the correct dependency graph, `pyproject.toml` also needs to be present License detection is not supported for `Poetry`. +By default, Trivy doesn't report development dependencies. Use the `--include-dev-deps` flag to include them. + + ### uv Trivy uses `uv.lock` to identify dependencies and find vulnerabilities. From 71743c1db3ca70063d4ab6a9f89a86371c967e9f Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Tue, 24 Dec 2024 14:17:25 +0600 Subject: [PATCH 3/5] refactor: rename functions Signed-off-by: nikpivkin --- pkg/fanal/analyzer/language/python/poetry/poetry.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/fanal/analyzer/language/python/poetry/poetry.go b/pkg/fanal/analyzer/language/python/poetry/poetry.go index 069d199cbfac..c9e2e1f6d8ac 100644 --- a/pkg/fanal/analyzer/language/python/poetry/poetry.go +++ b/pkg/fanal/analyzer/language/python/poetry/poetry.go @@ -104,9 +104,8 @@ func (a poetryAnalyzer) mergePyProject(fsys fs.FS, dir string, app *types.Applic return xerrors.Errorf("unable to parse %s: %w", path, err) } - prodRootDeps := project.Tool.Poetry.Dependencies - directDeps := prodRootDeps.Union(getDevDeps(project)) - prodDeps := getProdPackages(app, prodRootDeps) + directDeps := directDeps(project) + prodDeps := prodPackages(app, project.Tool.Poetry.Dependencies) // Identify the direct/transitive/dev dependencies for i, pkg := range app.Packages { @@ -121,15 +120,15 @@ func (a poetryAnalyzer) mergePyProject(fsys fs.FS, dir string, app *types.Applic return nil } -func getDevDeps(project pyproject.PyProject) set.Set[string] { - deps := set.New[string]() +func directDeps(project pyproject.PyProject) set.Set[string] { + deps := project.Tool.Poetry.Dependencies.Clone() for _, groupDeps := range project.Tool.Poetry.Groups { deps.Append(groupDeps.Dependencies.Items()...) } return deps } -func getProdPackages(app *types.Application, prodRootDeps set.Set[string]) set.Set[string] { +func prodPackages(app *types.Application, prodRootDeps set.Set[string]) set.Set[string] { packages := lo.SliceToMap(app.Packages, func(pkg types.Package) (string, types.Package) { return pkg.ID, pkg }) From 05bb7bd136d455016ca96403d06820ec457ccdf4 Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Tue, 24 Dec 2024 14:49:25 +0600 Subject: [PATCH 4/5] docs: link table and poetry section Signed-off-by: nikpivkin --- docs/docs/coverage/language/python.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/coverage/language/python.md b/docs/docs/coverage/language/python.md index 3033b2d71941..256509c04880 100644 --- a/docs/docs/coverage/language/python.md +++ b/docs/docs/coverage/language/python.md @@ -26,7 +26,7 @@ The following table provides an outline of the features Trivy offers. |-----------------|------------------|:-----------------------:|:----------------:|:------------------------------------:|:--------:|:----------------------------------------:| | pip | requirements.txt | - | Include | - | ✓ | ✓ | | Pipenv | Pipfile.lock | ✓ | Include | - | ✓ | Not needed | -| Poetry | poetry.lock | ✓ | Include | ✓ | - | Not needed | +| Poetry | poetry.lock | ✓ | [Include](#poetry) | ✓ | - | Not needed | | uv | uv.lock | ✓ | Exclude | ✓ | - | Not needed | From b5bafdab31fc752cec985035ecd181da351f21b0 Mon Sep 17 00:00:00 2001 From: nikpivkin Date: Tue, 24 Dec 2024 17:54:21 +0600 Subject: [PATCH 5/5] fix python docs Signed-off-by: nikpivkin --- docs/docs/coverage/language/python.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/coverage/language/python.md b/docs/docs/coverage/language/python.md index 256509c04880..4e1f8ddba2c1 100644 --- a/docs/docs/coverage/language/python.md +++ b/docs/docs/coverage/language/python.md @@ -26,7 +26,7 @@ The following table provides an outline of the features Trivy offers. |-----------------|------------------|:-----------------------:|:----------------:|:------------------------------------:|:--------:|:----------------------------------------:| | pip | requirements.txt | - | Include | - | ✓ | ✓ | | Pipenv | Pipfile.lock | ✓ | Include | - | ✓ | Not needed | -| Poetry | poetry.lock | ✓ | [Include](#poetry) | ✓ | - | Not needed | +| Poetry | poetry.lock | ✓ | [Exclude](#poetry) | ✓ | - | Not needed | | uv | uv.lock | ✓ | Exclude | ✓ | - | Not needed |