From 10a0cf407adb069d7da4f716bbbc781b1605c5e3 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 7 Jan 2025 14:30:45 +1300 Subject: [PATCH 1/7] test: add fixture --- .../fixtures/poetry/multiple-packages.v2.lock | 173 ++++++++++++++++++ pkg/lockfile/parse-poetry-lock_test.go | 75 ++++++++ 2 files changed, 248 insertions(+) create mode 100644 pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock diff --git a/pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock b/pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock new file mode 100644 index 0000000000..7f6575fdb1 --- /dev/null +++ b/pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock @@ -0,0 +1,173 @@ +# This file is automatically @generated by Poetry 2.0.0 and should not be changed by hand. + +[[package]] +name = "async-timeout" +version = "5.0.1" +description = "Timeout context manager for asyncio programs" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"caching\" and python_full_version < \"3.11.3\"" +files = [ + {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, + {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, +] + +[[package]] +name = "factory-boy" +version = "3.3.1" +description = "A versatile test fixtures replacement based on thoughtbot's factory_bot for Ruby." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "factory_boy-3.3.1-py2.py3-none-any.whl", hash = "sha256:7b1113c49736e1e9995bc2a18f4dbf2c52cf0f841103517010b1d825712ce3ca"}, + {file = "factory_boy-3.3.1.tar.gz", hash = "sha256:8317aa5289cdfc45f9cae570feb07a6177316c82e34d14df3c2e1f22f26abef0"}, +] + +[package.dependencies] +Faker = ">=0.7.0" + +[package.extras] +dev = ["Django", "Pillow", "SQLAlchemy", "coverage", "flake8", "isort", "mongoengine", "mongomock", "mypy", "tox", "wheel (>=0.32.0)", "zest.releaser[recommended]"] +doc = ["Sphinx", "sphinx-rtd-theme", "sphinxcontrib-spelling"] + +[[package]] +name = "faker" +version = "33.3.0" +description = "Faker is a Python package that generates fake data for you." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "Faker-33.3.0-py3-none-any.whl", hash = "sha256:ae074d9c7ef65817a93b448141a5531a16b2ea2e563dc5774578197c7c84060c"}, + {file = "faker-33.3.0.tar.gz", hash = "sha256:2abb551a05b75d268780b6095100a48afc43c53e97422002efbfc1272ebf5f26"}, +] + +[package.dependencies] +python-dateutil = ">=2.4" +typing-extensions = "*" + +[[package]] +name = "proto-plus" +version = "1.22.0" +description = "Beautiful, Pythonic protocol buffers." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "proto-plus-1.22.0.tar.gz", hash = "sha256:c2e6693fdf68c405a6428226915a8625d21d0513793598ae3287a1210478d8ec"}, + {file = "proto_plus-1.22.0-py3-none-any.whl", hash = "sha256:a27192d8cdc54e044f137b4c9053c9108cf5c065b46d067f1bcd389a911faf5b"}, +] + +[package.dependencies] +protobuf = ">=3.19.0,<5.0.0dev" + +[package.extras] +testing = ["google-api-core[grpc] (>=1.31.5)"] + +[[package]] +name = "protobuf" +version = "4.25.5" +description = "" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "protobuf-4.25.5-cp310-abi3-win32.whl", hash = "sha256:5e61fd921603f58d2f5acb2806a929b4675f8874ff5f330b7d6f7e2e784bbcd8"}, + {file = "protobuf-4.25.5-cp310-abi3-win_amd64.whl", hash = "sha256:4be0571adcbe712b282a330c6e89eae24281344429ae95c6d85e79e84780f5ea"}, + {file = "protobuf-4.25.5-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:b2fde3d805354df675ea4c7c6338c1aecd254dfc9925e88c6d31a2bcb97eb173"}, + {file = "protobuf-4.25.5-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:919ad92d9b0310070f8356c24b855c98df2b8bd207ebc1c0c6fcc9ab1e007f3d"}, + {file = "protobuf-4.25.5-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fe14e16c22be926d3abfcb500e60cab068baf10b542b8c858fa27e098123e331"}, + {file = "protobuf-4.25.5-cp38-cp38-win32.whl", hash = "sha256:98d8d8aa50de6a2747efd9cceba361c9034050ecce3e09136f90de37ddba66e1"}, + {file = "protobuf-4.25.5-cp38-cp38-win_amd64.whl", hash = "sha256:b0234dd5a03049e4ddd94b93400b67803c823cfc405689688f59b34e0742381a"}, + {file = "protobuf-4.25.5-cp39-cp39-win32.whl", hash = "sha256:abe32aad8561aa7cc94fc7ba4fdef646e576983edb94a73381b03c53728a626f"}, + {file = "protobuf-4.25.5-cp39-cp39-win_amd64.whl", hash = "sha256:7a183f592dc80aa7c8da7ad9e55091c4ffc9497b3054452d629bb85fa27c2a45"}, + {file = "protobuf-4.25.5-py3-none-any.whl", hash = "sha256:0aebecb809cae990f8129ada5ca273d9d670b76d9bfc9b1809f0a9c02b7dbf41"}, + {file = "protobuf-4.25.5.tar.gz", hash = "sha256:7f8249476b4a9473645db7f8ab42b02fe1488cbe5fb72fddd445e0665afd8584"}, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["dev"] +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "redis" +version = "5.2.1" +description = "Python client for Redis database and key-value store" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"caching\"" +files = [ + {file = "redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4"}, + {file = "redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f"}, +] + +[package.dependencies] +async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""} + +[package.extras] +hiredis = ["hiredis (>=3.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)"] + +[[package]] +name = "six" +version = "1.17.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main", "dev"] +files = [ + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[[package]] +name = "urllib3" +version = "2.3.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, + {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[extras] +caching = ["redis"] + +[metadata] +lock-version = "2.1" +python-versions = ">=3.9" +content-hash = "c62dd3f7cbbc0c98b7419c34c16af501899b577d018f76f51a99783d28eb7703" diff --git a/pkg/lockfile/parse-poetry-lock_test.go b/pkg/lockfile/parse-poetry-lock_test.go index 4b719e5d19..17eb171f86 100644 --- a/pkg/lockfile/parse-poetry-lock_test.go +++ b/pkg/lockfile/parse-poetry-lock_test.go @@ -210,3 +210,78 @@ func TestParsePoetryLock_OptionalPackage(t *testing.T) { }, }) } + +func TestParsePoetryLock_v2_MultiplePackages(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParsePoetryLock("fixtures/poetry/multiple-packages.v2.lock") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "async-timeout", + Version: "5.0.1", + Ecosystem: lockfile.PoetryEcosystem, + CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{"optional"}, + }, + { + Name: "factory-boy", + Version: "3.3.1", + Ecosystem: lockfile.PoetryEcosystem, + CompareAs: lockfile.PoetryEcosystem, + }, + { + Name: "faker", + Version: "33.3.0", + Ecosystem: lockfile.PoetryEcosystem, + CompareAs: lockfile.PoetryEcosystem, + }, + { + Name: "proto-plus", + Version: "1.22.0", + Ecosystem: lockfile.PoetryEcosystem, + CompareAs: lockfile.PoetryEcosystem, + }, + { + Name: "protobuf", + Version: "4.25.5", + Ecosystem: lockfile.PoetryEcosystem, + CompareAs: lockfile.PoetryEcosystem, + }, + { + Name: "python-dateutil", + Version: "2.9.0.post0", + Ecosystem: lockfile.PoetryEcosystem, + CompareAs: lockfile.PoetryEcosystem, + }, + { + Name: "six", + Version: "1.17.0", + Ecosystem: lockfile.PoetryEcosystem, + CompareAs: lockfile.PoetryEcosystem, + }, + { + Name: "typing-extensions", + Version: "4.12.2", + Ecosystem: lockfile.PoetryEcosystem, + CompareAs: lockfile.PoetryEcosystem, + }, + { + Name: "urllib3", + Version: "2.3.0", + Ecosystem: lockfile.PoetryEcosystem, + CompareAs: lockfile.PoetryEcosystem, + }, + { + Name: "redis", + Version: "5.2.1", + Ecosystem: lockfile.PoetryEcosystem, + CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{"optional"}, + }, + }) +} From e9333c34021bd243d73794de2dae45246f34c82c Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 7 Jan 2025 15:26:35 +1300 Subject: [PATCH 2/7] feat: support reading package groups for Poetry v2 lockfiles --- pkg/lockfile/parse-poetry-lock.go | 2 ++ pkg/lockfile/parse-poetry-lock_test.go | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/lockfile/parse-poetry-lock.go b/pkg/lockfile/parse-poetry-lock.go index ef5f70f45d..454ef75388 100644 --- a/pkg/lockfile/parse-poetry-lock.go +++ b/pkg/lockfile/parse-poetry-lock.go @@ -16,6 +16,7 @@ type PoetryLockPackage struct { Name string `toml:"name"` Version string `toml:"version"` Optional bool `toml:"optional"` + Groups []string `toml:"groups"` Source PoetryLockPackageSource `toml:"source"` } @@ -48,6 +49,7 @@ func (e PoetryLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { Name: lockPackage.Name, Version: lockPackage.Version, Commit: lockPackage.Source.Commit, + DepGroups: lockPackage.Groups, Ecosystem: PoetryEcosystem, CompareAs: PoetryEcosystem, } diff --git a/pkg/lockfile/parse-poetry-lock_test.go b/pkg/lockfile/parse-poetry-lock_test.go index 17eb171f86..288255801f 100644 --- a/pkg/lockfile/parse-poetry-lock_test.go +++ b/pkg/lockfile/parse-poetry-lock_test.go @@ -226,62 +226,70 @@ func TestParsePoetryLock_v2_MultiplePackages(t *testing.T) { Version: "5.0.1", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"optional"}, + DepGroups: []string{"main", "optional"}, }, { Name: "factory-boy", Version: "3.3.1", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{"dev"}, }, { Name: "faker", Version: "33.3.0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{"dev"}, }, { Name: "proto-plus", Version: "1.22.0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{"main"}, }, { Name: "protobuf", Version: "4.25.5", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{"main"}, }, { Name: "python-dateutil", Version: "2.9.0.post0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{"dev"}, }, { Name: "six", Version: "1.17.0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{"main", "dev"}, }, { Name: "typing-extensions", Version: "4.12.2", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{"dev"}, }, { Name: "urllib3", Version: "2.3.0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{"dev"}, }, { Name: "redis", Version: "5.2.1", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"optional"}, + DepGroups: []string{"main", "optional"}, }, }) } From 7397e28ec183f48b361d994a6c0e2064fc8891e0 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 7 Jan 2025 16:18:18 +1300 Subject: [PATCH 3/7] fix: omit the `main` group --- pkg/lockfile/parse-poetry-lock.go | 16 +++++++++++++++- pkg/lockfile/parse-poetry-lock_test.go | 10 +++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/pkg/lockfile/parse-poetry-lock.go b/pkg/lockfile/parse-poetry-lock.go index 454ef75388..0886b84cb3 100644 --- a/pkg/lockfile/parse-poetry-lock.go +++ b/pkg/lockfile/parse-poetry-lock.go @@ -33,6 +33,20 @@ func (e PoetryLockExtractor) ShouldExtract(path string) bool { return filepath.Base(path) == "poetry.lock" } +// removeMainGroup removes the "main" group from the list of groups, +// as it is the default group used by Poetry for dependencies +func removeMainGroup(groups []string) []string { + var g []string + + for _, group := range groups { + if group != "main" { + g = append(g, group) + } + } + + return g +} + func (e PoetryLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { var parsedLockfile *PoetryLockFile @@ -49,7 +63,7 @@ func (e PoetryLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { Name: lockPackage.Name, Version: lockPackage.Version, Commit: lockPackage.Source.Commit, - DepGroups: lockPackage.Groups, + DepGroups: removeMainGroup(lockPackage.Groups), Ecosystem: PoetryEcosystem, CompareAs: PoetryEcosystem, } diff --git a/pkg/lockfile/parse-poetry-lock_test.go b/pkg/lockfile/parse-poetry-lock_test.go index 288255801f..f014c5d154 100644 --- a/pkg/lockfile/parse-poetry-lock_test.go +++ b/pkg/lockfile/parse-poetry-lock_test.go @@ -226,7 +226,7 @@ func TestParsePoetryLock_v2_MultiplePackages(t *testing.T) { Version: "5.0.1", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"main", "optional"}, + DepGroups: []string{"optional"}, }, { Name: "factory-boy", @@ -247,14 +247,14 @@ func TestParsePoetryLock_v2_MultiplePackages(t *testing.T) { Version: "1.22.0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"main"}, + DepGroups: []string{}, }, { Name: "protobuf", Version: "4.25.5", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"main"}, + DepGroups: []string{}, }, { Name: "python-dateutil", @@ -268,7 +268,7 @@ func TestParsePoetryLock_v2_MultiplePackages(t *testing.T) { Version: "1.17.0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"main", "dev"}, + DepGroups: []string{"dev"}, }, { Name: "typing-extensions", @@ -289,7 +289,7 @@ func TestParsePoetryLock_v2_MultiplePackages(t *testing.T) { Version: "5.2.1", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"main", "optional"}, + DepGroups: []string{"optional"}, }, }) } From f7cf33125b5601a1e3b9a771813378b9839fd123 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 7 Jan 2025 16:22:13 +1300 Subject: [PATCH 4/7] test: improve fixture a little --- pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock | 10 +++++----- pkg/lockfile/parse-poetry-lock_test.go | 10 ++++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock b/pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock index 7f6575fdb1..307130ec1f 100644 --- a/pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock +++ b/pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock @@ -38,7 +38,7 @@ version = "33.3.0" description = "Faker is a Python package that generates fake data for you." optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["dev", "test"] files = [ {file = "Faker-33.3.0-py3-none-any.whl", hash = "sha256:ae074d9c7ef65817a93b448141a5531a16b2ea2e563dc5774578197c7c84060c"}, {file = "faker-33.3.0.tar.gz", hash = "sha256:2abb551a05b75d268780b6095100a48afc43c53e97422002efbfc1272ebf5f26"}, @@ -93,7 +93,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["dev"] +groups = ["dev", "test"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -128,7 +128,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main", "dev"] +groups = ["main", "dev", "test"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -140,7 +140,7 @@ version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["dev", "test"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -170,4 +170,4 @@ caching = ["redis"] [metadata] lock-version = "2.1" python-versions = ">=3.9" -content-hash = "c62dd3f7cbbc0c98b7419c34c16af501899b577d018f76f51a99783d28eb7703" +content-hash = "b80bca1ee02b055316dd15ab627b201152c1e83c2f95652fe4932716d8e8c46b" diff --git a/pkg/lockfile/parse-poetry-lock_test.go b/pkg/lockfile/parse-poetry-lock_test.go index f014c5d154..c916ac9b5a 100644 --- a/pkg/lockfile/parse-poetry-lock_test.go +++ b/pkg/lockfile/parse-poetry-lock_test.go @@ -240,42 +240,40 @@ func TestParsePoetryLock_v2_MultiplePackages(t *testing.T) { Version: "33.3.0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"dev"}, + DepGroups: []string{"dev", "test"}, }, { Name: "proto-plus", Version: "1.22.0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{}, }, { Name: "protobuf", Version: "4.25.5", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{}, }, { Name: "python-dateutil", Version: "2.9.0.post0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"dev"}, + DepGroups: []string{"dev", "test"}, }, { Name: "six", Version: "1.17.0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"dev"}, + DepGroups: []string{"dev", "test"}, }, { Name: "typing-extensions", Version: "4.12.2", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"dev"}, + DepGroups: []string{"dev", "test"}, }, { Name: "urllib3", From 15b23edaa35789d7c8fb0d2a015cc67dc72bcd40 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 7 Jan 2025 16:30:55 +1300 Subject: [PATCH 5/7] fix: remove all other groups to indicate production dependencies --- pkg/lockfile/parse-poetry-lock.go | 16 +++++++--------- pkg/lockfile/parse-poetry-lock_test.go | 4 +++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/lockfile/parse-poetry-lock.go b/pkg/lockfile/parse-poetry-lock.go index 0886b84cb3..85874f4a53 100644 --- a/pkg/lockfile/parse-poetry-lock.go +++ b/pkg/lockfile/parse-poetry-lock.go @@ -33,18 +33,16 @@ func (e PoetryLockExtractor) ShouldExtract(path string) bool { return filepath.Base(path) == "poetry.lock" } -// removeMainGroup removes the "main" group from the list of groups, -// as it is the default group used by Poetry for dependencies -func removeMainGroup(groups []string) []string { - var g []string - +func resolvePoetryPackageGroups(groups []string) []string { for _, group := range groups { - if group != "main" { - g = append(g, group) + // the "main" group is the default group used for "production" dependencies, + // which we represent by an empty slice aka no groups + if group == "main" { + return []string{} } } - return g + return groups } func (e PoetryLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { @@ -63,7 +61,7 @@ func (e PoetryLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { Name: lockPackage.Name, Version: lockPackage.Version, Commit: lockPackage.Source.Commit, - DepGroups: removeMainGroup(lockPackage.Groups), + DepGroups: resolvePoetryPackageGroups(lockPackage.Groups), Ecosystem: PoetryEcosystem, CompareAs: PoetryEcosystem, } diff --git a/pkg/lockfile/parse-poetry-lock_test.go b/pkg/lockfile/parse-poetry-lock_test.go index c916ac9b5a..ab373bd5c3 100644 --- a/pkg/lockfile/parse-poetry-lock_test.go +++ b/pkg/lockfile/parse-poetry-lock_test.go @@ -247,12 +247,14 @@ func TestParsePoetryLock_v2_MultiplePackages(t *testing.T) { Version: "1.22.0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{}, }, { Name: "protobuf", Version: "4.25.5", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{}, }, { Name: "python-dateutil", @@ -266,7 +268,7 @@ func TestParsePoetryLock_v2_MultiplePackages(t *testing.T) { Version: "1.17.0", Ecosystem: lockfile.PoetryEcosystem, CompareAs: lockfile.PoetryEcosystem, - DepGroups: []string{"dev", "test"}, + DepGroups: []string{}, }, { Name: "typing-extensions", From c4c08a9265642283e2a02985584d6ff12e32b0f6 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Wed, 8 Jan 2025 07:50:55 +1300 Subject: [PATCH 6/7] refactor: optimize Poetry grouping logic --- pkg/lockfile/parse-poetry-lock.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/pkg/lockfile/parse-poetry-lock.go b/pkg/lockfile/parse-poetry-lock.go index 85874f4a53..234120d6e7 100644 --- a/pkg/lockfile/parse-poetry-lock.go +++ b/pkg/lockfile/parse-poetry-lock.go @@ -33,8 +33,14 @@ func (e PoetryLockExtractor) ShouldExtract(path string) bool { return filepath.Base(path) == "poetry.lock" } -func resolvePoetryPackageGroups(groups []string) []string { - for _, group := range groups { +func resolvePoetryPackageGroups(pkg PoetryLockPackage) []string { + // by definition an optional package cannot be in any other group, + // otherwise that would make it a required package + if pkg.Optional { + return []string{"optional"} + } + + for _, group := range pkg.Groups { // the "main" group is the default group used for "production" dependencies, // which we represent by an empty slice aka no groups if group == "main" { @@ -42,7 +48,7 @@ func resolvePoetryPackageGroups(groups []string) []string { } } - return groups + return pkg.Groups } func (e PoetryLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { @@ -57,18 +63,14 @@ func (e PoetryLockExtractor) Extract(f DepFile) ([]PackageDetails, error) { packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) for _, lockPackage := range parsedLockfile.Packages { - pkgDetails := PackageDetails{ + packages = append(packages, PackageDetails{ Name: lockPackage.Name, Version: lockPackage.Version, Commit: lockPackage.Source.Commit, - DepGroups: resolvePoetryPackageGroups(lockPackage.Groups), + DepGroups: resolvePoetryPackageGroups(lockPackage), Ecosystem: PoetryEcosystem, CompareAs: PoetryEcosystem, - } - if lockPackage.Optional { - pkgDetails.DepGroups = append(pkgDetails.DepGroups, "optional") - } - packages = append(packages, pkgDetails) + }) } return packages, nil From c7b506f7f2b5425bc9253642b40450abf8269476 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Wed, 8 Jan 2025 08:34:31 +1300 Subject: [PATCH 7/7] test: update case to include a python version constraint --- .../fixtures/poetry/multiple-packages.v2.lock | 22 ++++++++++++++++++- pkg/lockfile/parse-poetry-lock_test.go | 7 ++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock b/pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock index 307130ec1f..e32494f235 100644 --- a/pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock +++ b/pkg/lockfile/fixtures/poetry/multiple-packages.v2.lock @@ -55,6 +55,7 @@ description = "Beautiful, Pythonic protocol buffers." optional = false python-versions = ">=3.6" groups = ["main"] +markers = "python_version < \"3.10\"" files = [ {file = "proto-plus-1.22.0.tar.gz", hash = "sha256:c2e6693fdf68c405a6428226915a8625d21d0513793598ae3287a1210478d8ec"}, {file = "proto_plus-1.22.0-py3-none-any.whl", hash = "sha256:a27192d8cdc54e044f137b4c9053c9108cf5c065b46d067f1bcd389a911faf5b"}, @@ -66,6 +67,25 @@ protobuf = ">=3.19.0,<5.0.0dev" [package.extras] testing = ["google-api-core[grpc] (>=1.31.5)"] +[[package]] +name = "proto-plus" +version = "1.23.0" +description = "Beautiful, Pythonic protocol buffers." +optional = false +python-versions = ">=3.6" +groups = ["main"] +markers = "python_version >= \"3.10\"" +files = [ + {file = "proto-plus-1.23.0.tar.gz", hash = "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2"}, + {file = "proto_plus-1.23.0-py3-none-any.whl", hash = "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c"}, +] + +[package.dependencies] +protobuf = ">=3.19.0,<5.0.0dev" + +[package.extras] +testing = ["google-api-core[grpc] (>=1.31.5)"] + [[package]] name = "protobuf" version = "4.25.5" @@ -170,4 +190,4 @@ caching = ["redis"] [metadata] lock-version = "2.1" python-versions = ">=3.9" -content-hash = "b80bca1ee02b055316dd15ab627b201152c1e83c2f95652fe4932716d8e8c46b" +content-hash = "ab8a643d59b9404536d1a035b0f7324ca1e2b906b5a69cf6ae8321a65efc3870" diff --git a/pkg/lockfile/parse-poetry-lock_test.go b/pkg/lockfile/parse-poetry-lock_test.go index ab373bd5c3..cac92fbe1b 100644 --- a/pkg/lockfile/parse-poetry-lock_test.go +++ b/pkg/lockfile/parse-poetry-lock_test.go @@ -249,6 +249,13 @@ func TestParsePoetryLock_v2_MultiplePackages(t *testing.T) { CompareAs: lockfile.PoetryEcosystem, DepGroups: []string{}, }, + { + Name: "proto-plus", + Version: "1.23.0", + Ecosystem: lockfile.PoetryEcosystem, + CompareAs: lockfile.PoetryEcosystem, + DepGroups: []string{}, + }, { Name: "protobuf", Version: "4.25.5",