From bdf0cacd4775cbe8f331408e6fe3937c9a7675ad Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Mon, 2 Dec 2024 07:39:41 +1300 Subject: [PATCH 01/10] feat: support extracting from `uv.lock` --- .../python/uvlock/testdata/empty.lock | 7 + .../uvlock/testdata/grouped-packages.lock | 220 ++++++++++++ .../python/uvlock/testdata/not-toml.txt | 1 + .../python/uvlock/testdata/one-package.lock | 22 ++ .../python/uvlock/testdata/two-packages.lock | 40 +++ .../language/python/uvlock/uvlock.go | 143 ++++++++ .../language/python/uvlock/uvlock_test.go | 334 ++++++++++++++++++ extractor/filesystem/list/list.go | 2 + extractor/filesystem/list/list_test.go | 6 +- 9 files changed, 772 insertions(+), 3 deletions(-) create mode 100644 extractor/filesystem/language/python/uvlock/testdata/empty.lock create mode 100644 extractor/filesystem/language/python/uvlock/testdata/grouped-packages.lock create mode 100644 extractor/filesystem/language/python/uvlock/testdata/not-toml.txt create mode 100644 extractor/filesystem/language/python/uvlock/testdata/one-package.lock create mode 100644 extractor/filesystem/language/python/uvlock/testdata/two-packages.lock create mode 100644 extractor/filesystem/language/python/uvlock/uvlock.go create mode 100644 extractor/filesystem/language/python/uvlock/uvlock_test.go diff --git a/extractor/filesystem/language/python/uvlock/testdata/empty.lock b/extractor/filesystem/language/python/uvlock/testdata/empty.lock new file mode 100644 index 00000000..7e42068a --- /dev/null +++ b/extractor/filesystem/language/python/uvlock/testdata/empty.lock @@ -0,0 +1,7 @@ +version = 1 +requires-python = ">=3.10" + +[[package]] +name = "uv-lockfiles" +version = "0.1.0" +source = { virtual = "." } diff --git a/extractor/filesystem/language/python/uvlock/testdata/grouped-packages.lock b/extractor/filesystem/language/python/uvlock/testdata/grouped-packages.lock new file mode 100644 index 00000000..3877bc7d --- /dev/null +++ b/extractor/filesystem/language/python/uvlock/testdata/grouped-packages.lock @@ -0,0 +1,220 @@ +version = 1 +requires-python = ">=3.10" + +[[package]] +name = "black" +version = "24.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d8/0d/cc2fb42b8c50d80143221515dd7e4766995bd07c56c9a3ed30baf080b6dc/black-24.10.0.tar.gz", hash = "sha256:846ea64c97afe3bc677b761787993be4991810ecc7a4a937816dd6bddedc4875", size = 645813 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/f3/465c0eb5cddf7dbbfe1fecd9b875d1dcf51b88923cd2c1d7e9ab95c6336b/black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812", size = 1623211 }, + { url = "https://files.pythonhosted.org/packages/df/57/b6d2da7d200773fdfcc224ffb87052cf283cec4d7102fab450b4a05996d8/black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea", size = 1457139 }, + { url = "https://files.pythonhosted.org/packages/6e/c5/9023b7673904a5188f9be81f5e129fff69f51f5515655fbd1d5a4e80a47b/black-24.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:649fff99a20bd06c6f727d2a27f401331dc0cc861fb69cde910fe95b01b5928f", size = 1753774 }, + { url = "https://files.pythonhosted.org/packages/e1/32/df7f18bd0e724e0d9748829765455d6643ec847b3f87e77456fc99d0edab/black-24.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:fe4d6476887de70546212c99ac9bd803d90b42fc4767f058a0baa895013fbb3e", size = 1414209 }, + { url = "https://files.pythonhosted.org/packages/c2/cc/7496bb63a9b06a954d3d0ac9fe7a73f3bf1cd92d7a58877c27f4ad1e9d41/black-24.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5a2221696a8224e335c28816a9d331a6c2ae15a2ee34ec857dcf3e45dbfa99ad", size = 1607468 }, + { url = "https://files.pythonhosted.org/packages/2b/e3/69a738fb5ba18b5422f50b4f143544c664d7da40f09c13969b2fd52900e0/black-24.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9da3333530dbcecc1be13e69c250ed8dfa67f43c4005fb537bb426e19200d50", size = 1437270 }, + { url = "https://files.pythonhosted.org/packages/c9/9b/2db8045b45844665c720dcfe292fdaf2e49825810c0103e1191515fc101a/black-24.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4007b1393d902b48b36958a216c20c4482f601569d19ed1df294a496eb366392", size = 1737061 }, + { url = "https://files.pythonhosted.org/packages/a3/95/17d4a09a5be5f8c65aa4a361444d95edc45def0de887810f508d3f65db7a/black-24.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:394d4ddc64782e51153eadcaaca95144ac4c35e27ef9b0a42e121ae7e57a9175", size = 1423293 }, + { url = "https://files.pythonhosted.org/packages/90/04/bf74c71f592bcd761610bbf67e23e6a3cff824780761f536512437f1e655/black-24.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e39e0fae001df40f95bd8cc36b9165c5e2ea88900167bddf258bacef9bbdc3", size = 1644256 }, + { url = "https://files.pythonhosted.org/packages/4c/ea/a77bab4cf1887f4b2e0bce5516ea0b3ff7d04ba96af21d65024629afedb6/black-24.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d37d422772111794b26757c5b55a3eade028aa3fde43121ab7b673d050949d65", size = 1448534 }, + { url = "https://files.pythonhosted.org/packages/4e/3e/443ef8bc1fbda78e61f79157f303893f3fddf19ca3c8989b163eb3469a12/black-24.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f", size = 1761892 }, + { url = "https://files.pythonhosted.org/packages/52/93/eac95ff229049a6901bc84fec6908a5124b8a0b7c26ea766b3b8a5debd22/black-24.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:30d2c30dc5139211dda799758559d1b049f7f14c580c409d6ad925b74a4208a8", size = 1434796 }, + { url = "https://files.pythonhosted.org/packages/d0/a0/a993f58d4ecfba035e61fca4e9f64a2ecae838fc9f33ab798c62173ed75c/black-24.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cbacacb19e922a1d75ef2b6ccaefcd6e93a2c05ede32f06a21386a04cedb981", size = 1643986 }, + { url = "https://files.pythonhosted.org/packages/37/d5/602d0ef5dfcace3fb4f79c436762f130abd9ee8d950fa2abdbf8bbc555e0/black-24.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f93102e0c5bb3907451063e08b9876dbeac810e7da5a8bfb7aeb5a9ef89066b", size = 1448085 }, + { url = "https://files.pythonhosted.org/packages/47/6d/a3a239e938960df1a662b93d6230d4f3e9b4a22982d060fc38c42f45a56b/black-24.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddacb691cdcdf77b96f549cf9591701d8db36b2f19519373d60d31746068dbf2", size = 1760928 }, + { url = "https://files.pythonhosted.org/packages/dd/cf/af018e13b0eddfb434df4d9cd1b2b7892bab119f7a20123e93f6910982e8/black-24.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:680359d932801c76d2e9c9068d05c6b107f2584b2a5b88831c83962eb9984c1b", size = 1436875 }, + { url = "https://files.pythonhosted.org/packages/8d/a7/4b27c50537ebca8bec139b872861f9d2bf501c5ec51fcf897cb924d9e264/black-24.10.0-py3-none-any.whl", hash = "sha256:3bb2b7a1f7b685f85b11fed1ef10f8a9148bceb49853e47a294a3dd963c1dd7d", size = 206898 }, +] + +[[package]] +name = "click" +version = "8.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "platform_system == 'Windows'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", size = 97941 }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + +[[package]] +name = "emoji" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/64/812d7e2ae0ac2ade0d6583f911f99240c80f700afbe8391df10e547f564d/emoji-2.14.0.tar.gz", hash = "sha256:f68ac28915a2221667cddb3e6c589303c3c6954c6c5af6fefaec7f9bdf72fdca", size = 593932 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/56/4ddf8b36aa4b52077045b17ffb8958f3360b250df4143d1482d9d5bb54d5/emoji-2.14.0-py3-none-any.whl", hash = "sha256:fcc936bf374b1aec67dda5303ae99710ba88cc9cdce2d1a71c5f2204e6d78799", size = 586897 }, +] + +[[package]] +name = "flake8" +version = "7.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mccabe" }, + { name = "pycodestyle" }, + { name = "pyflakes" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/72/e8d66150c4fcace3c0a450466aa3480506ba2cae7b61e100a2613afc3907/flake8-7.1.1.tar.gz", hash = "sha256:049d058491e228e03e67b390f311bbf88fce2dbaa8fa673e7aea87b7198b8d38", size = 48054 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/42/65004373ac4617464f35ed15931b30d764f53cdd30cc78d5aea349c8c050/flake8-7.1.1-py2.py3-none-any.whl", hash = "sha256:597477df7860daa5aa0fdd84bf5208a043ab96b8e96ab708770ae0364dd03213", size = 57731 }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350 }, +] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/a4/1ab47638b92648243faf97a5aeb6ea83059cc3624972ab6b8d2316078d3f/mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782", size = 4433 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d", size = 4695 }, +] + +[[package]] +name = "packaging" +version = "24.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/63/68dbb6eb2de9cb10ee4c9c14a0148804425e13c4fb20d61cce69f53106da/packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f", size = 163950 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451 }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191 }, +] + +[[package]] +name = "platformdirs" +version = "4.3.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/fc/128cc9cb8f03208bdbf93d3aa862e16d376844a14f9a0ce5cf4507372de4/platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907", size = 21302 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb", size = 18439 }, +] + +[[package]] +name = "pycodestyle" +version = "2.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/aa/210b2c9aedd8c1cbeea31a50e42050ad56187754b34eb214c46709445801/pycodestyle-2.12.1.tar.gz", hash = "sha256:6838eae08bbce4f6accd5d5572075c63626a15ee3e6f842df996bf62f6d73521", size = 39232 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/d8/a211b3f85e99a0daa2ddec96c949cac6824bd305b040571b82a03dd62636/pycodestyle-2.12.1-py2.py3-none-any.whl", hash = "sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3", size = 31284 }, +] + +[[package]] +name = "pyflakes" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/f9/669d8c9c86613c9d568757c7f5824bd3197d7b1c6c27553bc5618a27cce2/pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f", size = 63788 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/d7/f1b7db88d8e4417c5d47adad627a93547f44bdc9028372dbd2313f34a855/pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a", size = 62725 }, +] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077 }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429 }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067 }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030 }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898 }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894 }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319 }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273 }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310 }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309 }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762 }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453 }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486 }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349 }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159 }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243 }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645 }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584 }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875 }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418 }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708 }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582 }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543 }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691 }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170 }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530 }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666 }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954 }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724 }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383 }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257 }, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 }, +] + +[[package]] +name = "uv-lockfiles" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "emoji" }, +] + +[package.optional-dependencies] +cli = [ + { name = "click" }, +] +dev = [ + { name = "black" }, +] +test = [ + { name = "black" }, + { name = "flake8" }, +] + +[package.metadata] +requires-dist = [ + { name = "black", marker = "extra == 'dev'" }, + { name = "black", marker = "extra == 'test'" }, + { name = "click", marker = "extra == 'cli'" }, + { name = "emoji" }, + { name = "flake8", marker = "extra == 'test'" }, +] diff --git a/extractor/filesystem/language/python/uvlock/testdata/not-toml.txt b/extractor/filesystem/language/python/uvlock/testdata/not-toml.txt new file mode 100644 index 00000000..ddd3cb93 --- /dev/null +++ b/extractor/filesystem/language/python/uvlock/testdata/not-toml.txt @@ -0,0 +1 @@ +this is not valid toml! (I think) diff --git a/extractor/filesystem/language/python/uvlock/testdata/one-package.lock b/extractor/filesystem/language/python/uvlock/testdata/one-package.lock new file mode 100644 index 00000000..51d644fc --- /dev/null +++ b/extractor/filesystem/language/python/uvlock/testdata/one-package.lock @@ -0,0 +1,22 @@ +version = 1 +requires-python = ">=3.10" + +[[package]] +name = "emoji" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/64/812d7e2ae0ac2ade0d6583f911f99240c80f700afbe8391df10e547f564d/emoji-2.14.0.tar.gz", hash = "sha256:f68ac28915a2221667cddb3e6c589303c3c6954c6c5af6fefaec7f9bdf72fdca", size = 593932 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/56/4ddf8b36aa4b52077045b17ffb8958f3360b250df4143d1482d9d5bb54d5/emoji-2.14.0-py3-none-any.whl", hash = "sha256:fcc936bf374b1aec67dda5303ae99710ba88cc9cdce2d1a71c5f2204e6d78799", size = 586897 }, +] + +[[package]] +name = "uv-lockfiles" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "emoji" }, +] + +[package.metadata] +requires-dist = [{ name = "emoji" }] diff --git a/extractor/filesystem/language/python/uvlock/testdata/two-packages.lock b/extractor/filesystem/language/python/uvlock/testdata/two-packages.lock new file mode 100644 index 00000000..5956baa1 --- /dev/null +++ b/extractor/filesystem/language/python/uvlock/testdata/two-packages.lock @@ -0,0 +1,40 @@ +version = 1 +requires-python = ">=3.10" + +[[package]] +name = "emoji" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/64/812d7e2ae0ac2ade0d6583f911f99240c80f700afbe8391df10e547f564d/emoji-2.14.0.tar.gz", hash = "sha256:f68ac28915a2221667cddb3e6c589303c3c6954c6c5af6fefaec7f9bdf72fdca", size = 593932 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/56/4ddf8b36aa4b52077045b17ffb8958f3360b250df4143d1482d9d5bb54d5/emoji-2.14.0-py3-none-any.whl", hash = "sha256:fcc936bf374b1aec67dda5303ae99710ba88cc9cdce2d1a71c5f2204e6d78799", size = 586897 }, +] + +[[package]] +name = "protobuf" +version = "4.25.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/67/dd/48d5fdb68ec74d70fabcc252e434492e56f70944d9f17b6a15e3746d2295/protobuf-4.25.5.tar.gz", hash = "sha256:7f8249476b4a9473645db7f8ab42b02fe1488cbe5fb72fddd445e0665afd8584", size = 380315 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/35/1b3c5a5e6107859c4ca902f4fbb762e48599b78129a05d20684fef4a4d04/protobuf-4.25.5-cp310-abi3-win32.whl", hash = "sha256:5e61fd921603f58d2f5acb2806a929b4675f8874ff5f330b7d6f7e2e784bbcd8", size = 392457 }, + { url = "https://files.pythonhosted.org/packages/a7/ad/bf3f358e90b7e70bf7fb520702cb15307ef268262292d3bdb16ad8ebc815/protobuf-4.25.5-cp310-abi3-win_amd64.whl", hash = "sha256:4be0571adcbe712b282a330c6e89eae24281344429ae95c6d85e79e84780f5ea", size = 413449 }, + { url = "https://files.pythonhosted.org/packages/51/49/d110f0a43beb365758a252203c43eaaad169fe7749da918869a8c991f726/protobuf-4.25.5-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:b2fde3d805354df675ea4c7c6338c1aecd254dfc9925e88c6d31a2bcb97eb173", size = 394248 }, + { url = "https://files.pythonhosted.org/packages/c6/ab/0f384ca0bc6054b1a7b6009000ab75d28a5506e4459378b81280ae7fd358/protobuf-4.25.5-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:919ad92d9b0310070f8356c24b855c98df2b8bd207ebc1c0c6fcc9ab1e007f3d", size = 293717 }, + { url = "https://files.pythonhosted.org/packages/05/a6/094a2640be576d760baa34c902dcb8199d89bce9ed7dd7a6af74dcbbd62d/protobuf-4.25.5-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fe14e16c22be926d3abfcb500e60cab068baf10b542b8c858fa27e098123e331", size = 294635 }, + { url = "https://files.pythonhosted.org/packages/33/90/f198a61df8381fb43ae0fe81b3d2718e8dcc51ae8502c7657ab9381fbc4f/protobuf-4.25.5-py3-none-any.whl", hash = "sha256:0aebecb809cae990f8129ada5ca273d9d670b76d9bfc9b1809f0a9c02b7dbf41", size = 156467 }, +] + +[[package]] +name = "uv-lockfiles" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "emoji" }, + { name = "protobuf" }, +] + +[package.metadata] +requires-dist = [ + { name = "emoji" }, + { name = "protobuf", specifier = ">=3.19.0,<5.0.0.dev0" }, +] diff --git a/extractor/filesystem/language/python/uvlock/uvlock.go b/extractor/filesystem/language/python/uvlock/uvlock.go new file mode 100644 index 00000000..bdebf2fb --- /dev/null +++ b/extractor/filesystem/language/python/uvlock/uvlock.go @@ -0,0 +1,143 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package uvlock extracts uv.lock files. +package uvlock + +import ( + "context" + "fmt" + "path/filepath" + "sort" + + "github.com/BurntSushi/toml" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/internal/pypipurl" + "github.com/google/osv-scalibr/extractor/filesystem/osv" + "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/purl" +) + +type uvLockPackageSource struct { + Virtual string `toml:"virtual"` + Registry string `toml:"registry"` +} + +type uvLockPackage struct { + Name string `toml:"name"` + Version string `toml:"version"` + Source uvLockPackageSource `toml:"source"` + + // uv stores "groups" as a table under "package" after all the packages, which due + // to how TOML works means it ends up being a property on the last package, even + // through in this context it's a global property rather than being per-package + Groups map[string][]uvOptionalDependency `toml:"optional-dependencies"` +} + +type uvOptionalDependency struct { + Name string `toml:"name"` +} +type uvLockFile struct { + Version int `toml:"version"` + Packages []uvLockPackage `toml:"package"` + Groups map[string][]uvOptionalDependency `toml:"package.optional-dependencies"` +} + +// Extractor extracts python packages from uv.lock files. +type Extractor struct{} + +// Name of the extractor +func (e Extractor) Name() string { return "python/uvlock" } + +// Version of the extractor +func (e Extractor) Version() int { return 0 } + +// Requirements of the extractor +func (e Extractor) Requirements() *plugin.Capabilities { + return &plugin.Capabilities{} +} + +// FileRequired returns true if the specified file matches uv lockfile patterns +func (e Extractor) FileRequired(api filesystem.FileAPI) bool { + return filepath.Base(api.Path()) == "uv.lock" +} + +// Extract extracts packages from uv.lock files passed through the scan input. +func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) { + var parsedLockfile *uvLockFile + + _, err := toml.NewDecoder(input.Reader).Decode(&parsedLockfile) + + if err != nil { + return []*extractor.Inventory{}, fmt.Errorf("could not extract from %s: %w", input.Path, err) + } + + packages := make([]*extractor.Inventory, 0, len(parsedLockfile.Packages)) + + var groups map[string][]uvOptionalDependency + + // uv stores "groups" as a table under "package" after all the packages, which due + // to how TOML works means it ends up being a property on the last package, even + // through in this context it's a global property rather than being per-package + if len(parsedLockfile.Packages) > 0 { + groups = parsedLockfile.Packages[len(parsedLockfile.Packages)-1].Groups + } + + for _, lockPackage := range parsedLockfile.Packages { + if lockPackage.Source.Virtual == "." { + continue + } + + pkgDetails := &extractor.Inventory{ + Name: lockPackage.Name, + Version: lockPackage.Version, + Locations: []string{input.Path}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + } + + depGroupVals := []string{} + + for group, deps := range groups { + for _, dep := range deps { + if dep.Name == lockPackage.Name { + depGroupVals = append(depGroupVals, group) + } + } + } + + sort.Strings(depGroupVals) + + pkgDetails.Metadata = osv.DepGroupMetadata{ + DepGroupVals: depGroupVals, + } + packages = append(packages, pkgDetails) + } + + return packages, nil +} + +// ToPURL converts an inventory created by this extractor into a PURL. +func (e Extractor) ToPURL(i *extractor.Inventory) *purl.PackageURL { + return pypipurl.MakePackageURL(i) +} + +// Ecosystem returns the OSV ecosystem ('PyPI') of the software extracted by this extractor. +func (e Extractor) Ecosystem(i *extractor.Inventory) string { + return "PyPI" +} + +var _ filesystem.Extractor = Extractor{} diff --git a/extractor/filesystem/language/python/uvlock/uvlock_test.go b/extractor/filesystem/language/python/uvlock/uvlock_test.go new file mode 100644 index 00000000..9aee7d1f --- /dev/null +++ b/extractor/filesystem/language/python/uvlock/uvlock_test.go @@ -0,0 +1,334 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package uvlock_test + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/uvlock" + "github.com/google/osv-scalibr/extractor/filesystem/osv" + "github.com/google/osv-scalibr/extractor/filesystem/simplefileapi" + "github.com/google/osv-scalibr/testing/extracttest" +) + +func TestExtractor_FileRequired(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + inputPath string + want bool + }{ + { + name: "", + inputPath: "", + want: false, + }, + { + name: "", + inputPath: "uv.lock", + want: true, + }, + { + name: "", + inputPath: "path/to/my/uv.lock", + want: true, + }, + { + name: "", + inputPath: "path/to/my/uv.lock/file", + want: false, + }, + { + name: "", + inputPath: "path/to/my/uv.lock.file", + want: false, + }, + { + name: "", + inputPath: "path.to.my.uv.lock", + want: false, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + e := uvlock.Extractor{} + got := e.FileRequired(simplefileapi.New(tt.inputPath, nil)) + if got != tt.want { + t.Errorf("FileRequired(%q, FileInfo) got = %v, want %v", tt.inputPath, got, tt.want) + } + }) + } +} + +func TestExtractor_Extract(t *testing.T) { + t.Parallel() + tests := []extracttest.TestTableEntry{ + { + Name: "invalid toml", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/not-toml.txt", + }, + WantErr: extracttest.ContainsErrStr{Str: "could not extract from"}, + WantInventory: []*extractor.Inventory{}, + }, + { + Name: "no packages", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/empty.lock", + }, + WantInventory: []*extractor.Inventory{}, + }, + { + Name: "one package", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/one-package.lock", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "emoji", + Version: "2.14.0", + Locations: []string{"testdata/one-package.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "two packages", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/two-packages.lock", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "emoji", + Version: "2.14.0", + Locations: []string{"testdata/two-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "protobuf", + Version: "4.25.5", + Locations: []string{"testdata/two-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + { + Name: "grouped packages", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/grouped-packages.lock", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "emoji", + Version: "2.14.0", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "click", + Version: "8.1.7", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"cli"}, + }, + }, + { + Name: "colorama", + Version: "0.4.6", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "black", + Version: "24.10.0", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"dev", "test"}, + }, + }, + { + Name: "flake8", + Version: "7.1.1", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{"test"}, + }, + }, + { + Name: "mccabe", + Version: "0.7.0", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "mypy-extensions", + Version: "1.0.0", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "packaging", + Version: "24.2", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "pathspec", + Version: "0.12.1", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "platformdirs", + Version: "4.3.6", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "pycodestyle", + Version: "2.12.1", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "pyflakes", + Version: "3.2.0", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "tomli", + Version: "2.2.1", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + { + Name: "typing-extensions", + Version: "4.12.2", + Locations: []string{"testdata/grouped-packages.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.Name, func(t *testing.T) { + t.Parallel() + extr := uvlock.Extractor{} + + scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig) + defer extracttest.CloseTestScanInput(t, scanInput) + + got, err := extr.Extract(context.Background(), &scanInput) + + if diff := cmp.Diff(tt.WantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s.Extract(%q) error diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + return + } + + if diff := cmp.Diff(tt.WantInventory, got, cmpopts.SortSlices(extracttest.InventoryCmpLess)); diff != "" { + t.Errorf("%s.Extract(%q) diff (-want +got):\n%s", extr.Name(), tt.InputConfig.Path, diff) + } + }) + } +} diff --git a/extractor/filesystem/list/list.go b/extractor/filesystem/list/list.go index c04c01b6..4b754224 100644 --- a/extractor/filesystem/list/list.go +++ b/extractor/filesystem/list/list.go @@ -51,6 +51,7 @@ import ( "github.com/google/osv-scalibr/extractor/filesystem/language/python/pipfilelock" "github.com/google/osv-scalibr/extractor/filesystem/language/python/poetrylock" "github.com/google/osv-scalibr/extractor/filesystem/language/python/requirements" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/uvlock" "github.com/google/osv-scalibr/extractor/filesystem/language/python/wheelegg" "github.com/google/osv-scalibr/extractor/filesystem/language/r/renvlock" "github.com/google/osv-scalibr/extractor/filesystem/language/ruby/gemfilelock" @@ -105,6 +106,7 @@ var ( pdmlock.Extractor{}, poetrylock.Extractor{}, condameta.Extractor{}, + uvlock.Extractor{}, } // Go extractors. Go []filesystem.Extractor = []filesystem.Extractor{ diff --git a/extractor/filesystem/list/list_test.go b/extractor/filesystem/list/list_test.go index f6e8ca4c..af208e9b 100644 --- a/extractor/filesystem/list/list_test.go +++ b/extractor/filesystem/list/list_test.go @@ -69,17 +69,17 @@ func TestExtractorsFromNames(t *testing.T) { { desc: "Find all extractors of a type", names: []string{"python"}, - wantExts: []string{"python/pdmlock", "python/Pipfilelock", "python/poetrylock", "python/condameta", "python/wheelegg", "python/requirements"}, + wantExts: []string{"python/pdmlock", "python/Pipfilelock", "python/poetrylock", "python/condameta", "python/uvlock", "python/wheelegg", "python/requirements"}, }, { desc: "Case-insensitive", names: []string{"Python"}, - wantExts: []string{"python/pdmlock", "python/Pipfilelock", "python/poetrylock", "python/condameta", "python/wheelegg", "python/requirements"}, + wantExts: []string{"python/pdmlock", "python/Pipfilelock", "python/poetrylock", "python/condameta", "python/uvlock", "python/wheelegg", "python/requirements"}, }, { desc: "Remove duplicates", names: []string{"python", "python"}, - wantExts: []string{"python/pdmlock", "python/Pipfilelock", "python/poetrylock", "python/condameta", "python/wheelegg", "python/requirements"}, + wantExts: []string{"python/pdmlock", "python/Pipfilelock", "python/poetrylock", "python/condameta", "python/uvlock", "python/wheelegg", "python/requirements"}, }, { desc: "Nonexistent plugin", From f2280aabb6d5753ff400ecef2545372452beb9a8 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Mon, 2 Dec 2024 09:03:27 +1300 Subject: [PATCH 02/10] fix: support parsing commits from Git based sources --- .../python/uvlock/testdata/source-git.lock | 18 ++++++++++++++++++ .../language/python/uvlock/uvlock.go | 6 +++++- .../language/python/uvlock/uvlock_test.go | 19 +++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 extractor/filesystem/language/python/uvlock/testdata/source-git.lock diff --git a/extractor/filesystem/language/python/uvlock/testdata/source-git.lock b/extractor/filesystem/language/python/uvlock/testdata/source-git.lock new file mode 100644 index 00000000..cb9bd49d --- /dev/null +++ b/extractor/filesystem/language/python/uvlock/testdata/source-git.lock @@ -0,0 +1,18 @@ +version = 1 +requires-python = ">=3.10" + +[[package]] +name = "ruff" +version = "0.8.1" +source = { git = "https://github.com/astral-sh/ruff#84748be16341b76e073d117329f7f5f4ee2941ad" } + +[[package]] +name = "uv-lockfiles" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [{ name = "ruff", git = "https://github.com/astral-sh/ruff" }] diff --git a/extractor/filesystem/language/python/uvlock/uvlock.go b/extractor/filesystem/language/python/uvlock/uvlock.go index bdebf2fb..c830e688 100644 --- a/extractor/filesystem/language/python/uvlock/uvlock.go +++ b/extractor/filesystem/language/python/uvlock/uvlock.go @@ -20,6 +20,7 @@ import ( "fmt" "path/filepath" "sort" + "strings" "github.com/BurntSushi/toml" "github.com/google/osv-scalibr/extractor" @@ -32,6 +33,7 @@ import ( type uvLockPackageSource struct { Virtual string `toml:"virtual"` + Git string `toml:"git"` Registry string `toml:"registry"` } @@ -100,12 +102,14 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] continue } + _, commit, _ := strings.Cut(lockPackage.Source.Git, "#") + pkgDetails := &extractor.Inventory{ Name: lockPackage.Name, Version: lockPackage.Version, Locations: []string{input.Path}, SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", + Commit: commit, }, } diff --git a/extractor/filesystem/language/python/uvlock/uvlock_test.go b/extractor/filesystem/language/python/uvlock/uvlock_test.go index 9aee7d1f..865e5887 100644 --- a/extractor/filesystem/language/python/uvlock/uvlock_test.go +++ b/extractor/filesystem/language/python/uvlock/uvlock_test.go @@ -146,6 +146,25 @@ func TestExtractor_Extract(t *testing.T) { }, }, }, + { + Name: "source git", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/source-git.lock", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "ruff", + Version: "0.8.1", + Locations: []string{"testdata/source-git.lock"}, + SourceCode: &extractor.SourceCodeIdentifier{ + Commit: "84748be16341b76e073d117329f7f5f4ee2941ad", + }, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + }, + }, + }, { Name: "grouped packages", InputConfig: extracttest.ScanInputMockConfig{ From 155f9663c6afb643dc8fb5c54d9182c5ff3edfb5 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Wed, 4 Dec 2024 07:15:08 +1300 Subject: [PATCH 03/10] chore: remove top-level `Groups` field --- extractor/filesystem/language/python/uvlock/uvlock.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/extractor/filesystem/language/python/uvlock/uvlock.go b/extractor/filesystem/language/python/uvlock/uvlock.go index c830e688..a720fcc9 100644 --- a/extractor/filesystem/language/python/uvlock/uvlock.go +++ b/extractor/filesystem/language/python/uvlock/uvlock.go @@ -52,9 +52,8 @@ type uvOptionalDependency struct { Name string `toml:"name"` } type uvLockFile struct { - Version int `toml:"version"` - Packages []uvLockPackage `toml:"package"` - Groups map[string][]uvOptionalDependency `toml:"package.optional-dependencies"` + Version int `toml:"version"` + Packages []uvLockPackage `toml:"package"` } // Extractor extracts python packages from uv.lock files. From 7b1a299a586407b8349a5fd6270ff16424ac5322 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Wed, 4 Dec 2024 07:19:01 +1300 Subject: [PATCH 04/10] chore: add comment about why we're skipping virtual packages --- extractor/filesystem/language/python/uvlock/uvlock.go | 1 + 1 file changed, 1 insertion(+) diff --git a/extractor/filesystem/language/python/uvlock/uvlock.go b/extractor/filesystem/language/python/uvlock/uvlock.go index a720fcc9..0bb6a805 100644 --- a/extractor/filesystem/language/python/uvlock/uvlock.go +++ b/extractor/filesystem/language/python/uvlock/uvlock.go @@ -97,6 +97,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] } for _, lockPackage := range parsedLockfile.Packages { + // skip including the root "package", since its name and version are most likely arbitrary if lockPackage.Source.Virtual == "." { continue } From 548bd1b5faea5c3552fed2e24c3af344a3ee661f Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Wed, 4 Dec 2024 07:23:43 +1300 Subject: [PATCH 05/10] test: add cases for when file is valid-but-empty toml and has no `package`s at all --- .../language/python/uvlock/testdata/empty.lock | 7 ------- .../python/uvlock/testdata/no-dependencies.lock | 7 +++++++ .../python/uvlock/testdata/no-packages.lock | 2 ++ .../language/python/uvlock/uvlock_test.go | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 extractor/filesystem/language/python/uvlock/testdata/no-dependencies.lock create mode 100644 extractor/filesystem/language/python/uvlock/testdata/no-packages.lock diff --git a/extractor/filesystem/language/python/uvlock/testdata/empty.lock b/extractor/filesystem/language/python/uvlock/testdata/empty.lock index 7e42068a..e69de29b 100644 --- a/extractor/filesystem/language/python/uvlock/testdata/empty.lock +++ b/extractor/filesystem/language/python/uvlock/testdata/empty.lock @@ -1,7 +0,0 @@ -version = 1 -requires-python = ">=3.10" - -[[package]] -name = "uv-lockfiles" -version = "0.1.0" -source = { virtual = "." } diff --git a/extractor/filesystem/language/python/uvlock/testdata/no-dependencies.lock b/extractor/filesystem/language/python/uvlock/testdata/no-dependencies.lock new file mode 100644 index 00000000..7e42068a --- /dev/null +++ b/extractor/filesystem/language/python/uvlock/testdata/no-dependencies.lock @@ -0,0 +1,7 @@ +version = 1 +requires-python = ">=3.10" + +[[package]] +name = "uv-lockfiles" +version = "0.1.0" +source = { virtual = "." } diff --git a/extractor/filesystem/language/python/uvlock/testdata/no-packages.lock b/extractor/filesystem/language/python/uvlock/testdata/no-packages.lock new file mode 100644 index 00000000..66cf288a --- /dev/null +++ b/extractor/filesystem/language/python/uvlock/testdata/no-packages.lock @@ -0,0 +1,2 @@ +version = 1 +requires-python = ">=3.10" diff --git a/extractor/filesystem/language/python/uvlock/uvlock_test.go b/extractor/filesystem/language/python/uvlock/uvlock_test.go index 865e5887..d7a1a3a5 100644 --- a/extractor/filesystem/language/python/uvlock/uvlock_test.go +++ b/extractor/filesystem/language/python/uvlock/uvlock_test.go @@ -90,6 +90,20 @@ func TestExtractor_Extract(t *testing.T) { WantErr: extracttest.ContainsErrStr{Str: "could not extract from"}, WantInventory: []*extractor.Inventory{}, }, + { + Name: "empty file", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/empty.lock", + }, + WantInventory: []*extractor.Inventory{}, + }, + { + Name: "no dependencies", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/empty.lock", + }, + WantInventory: []*extractor.Inventory{}, + }, { Name: "no packages", InputConfig: extracttest.ScanInputMockConfig{ From b3c5c92746efebad352d5d111c0608074eb5b63b Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 5 Dec 2024 07:24:03 +1300 Subject: [PATCH 06/10] test: don't run in parallel --- extractor/filesystem/language/python/uvlock/uvlock_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/extractor/filesystem/language/python/uvlock/uvlock_test.go b/extractor/filesystem/language/python/uvlock/uvlock_test.go index d7a1a3a5..09307662 100644 --- a/extractor/filesystem/language/python/uvlock/uvlock_test.go +++ b/extractor/filesystem/language/python/uvlock/uvlock_test.go @@ -28,8 +28,6 @@ import ( ) func TestExtractor_FileRequired(t *testing.T) { - t.Parallel() - tests := []struct { name string inputPath string @@ -69,7 +67,6 @@ func TestExtractor_FileRequired(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - t.Parallel() e := uvlock.Extractor{} got := e.FileRequired(simplefileapi.New(tt.inputPath, nil)) if got != tt.want { @@ -80,7 +77,6 @@ func TestExtractor_FileRequired(t *testing.T) { } func TestExtractor_Extract(t *testing.T) { - t.Parallel() tests := []extracttest.TestTableEntry{ { Name: "invalid toml", @@ -346,7 +342,6 @@ func TestExtractor_Extract(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.Name, func(t *testing.T) { - t.Parallel() extr := uvlock.Extractor{} scanInput := extracttest.GenerateScanInputMock(t, tt.InputConfig) From c3058a5a60af4b8e408b1d9e9025a1d2801067f4 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 5 Dec 2024 07:24:15 +1300 Subject: [PATCH 07/10] test: remove unneeded loop copying --- extractor/filesystem/language/python/uvlock/uvlock_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/extractor/filesystem/language/python/uvlock/uvlock_test.go b/extractor/filesystem/language/python/uvlock/uvlock_test.go index 09307662..3c503891 100644 --- a/extractor/filesystem/language/python/uvlock/uvlock_test.go +++ b/extractor/filesystem/language/python/uvlock/uvlock_test.go @@ -65,7 +65,6 @@ func TestExtractor_FileRequired(t *testing.T) { }, } for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { e := uvlock.Extractor{} got := e.FileRequired(simplefileapi.New(tt.inputPath, nil)) @@ -340,7 +339,6 @@ func TestExtractor_Extract(t *testing.T) { } for _, tt := range tests { - tt := tt t.Run(tt.Name, func(t *testing.T) { extr := uvlock.Extractor{} From 899c5034f69e54dd708bd02cf0d371d9e8b94505 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 5 Dec 2024 07:28:22 +1300 Subject: [PATCH 08/10] fix: don't set `SourceCode` field unless there is actually a commit --- .../language/python/uvlock/uvlock.go | 7 ++- .../language/python/uvlock/uvlock_test.go | 51 ------------------- 2 files changed, 5 insertions(+), 53 deletions(-) diff --git a/extractor/filesystem/language/python/uvlock/uvlock.go b/extractor/filesystem/language/python/uvlock/uvlock.go index 0bb6a805..8d69659b 100644 --- a/extractor/filesystem/language/python/uvlock/uvlock.go +++ b/extractor/filesystem/language/python/uvlock/uvlock.go @@ -108,9 +108,12 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] Name: lockPackage.Name, Version: lockPackage.Version, Locations: []string{input.Path}, - SourceCode: &extractor.SourceCodeIdentifier{ + } + + if commit != "" { + pkgDetails.SourceCode = &extractor.SourceCodeIdentifier{ Commit: commit, - }, + } } depGroupVals := []string{} diff --git a/extractor/filesystem/language/python/uvlock/uvlock_test.go b/extractor/filesystem/language/python/uvlock/uvlock_test.go index 3c503891..b4ed1116 100644 --- a/extractor/filesystem/language/python/uvlock/uvlock_test.go +++ b/extractor/filesystem/language/python/uvlock/uvlock_test.go @@ -116,9 +116,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "emoji", Version: "2.14.0", Locations: []string{"testdata/one-package.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -135,9 +132,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "emoji", Version: "2.14.0", Locations: []string{"testdata/two-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -146,9 +140,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "protobuf", Version: "4.25.5", Locations: []string{"testdata/two-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -184,9 +175,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "emoji", Version: "2.14.0", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -195,9 +183,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "click", Version: "8.1.7", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{"cli"}, }, @@ -206,9 +191,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "colorama", Version: "0.4.6", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -217,9 +199,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "black", Version: "24.10.0", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{"dev", "test"}, }, @@ -228,9 +207,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "flake8", Version: "7.1.1", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{"test"}, }, @@ -239,9 +215,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "mccabe", Version: "0.7.0", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -250,9 +223,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "mypy-extensions", Version: "1.0.0", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -261,9 +231,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "packaging", Version: "24.2", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -272,9 +239,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "pathspec", Version: "0.12.1", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -283,9 +247,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "platformdirs", Version: "4.3.6", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -294,9 +255,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "pycodestyle", Version: "2.12.1", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -305,9 +263,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "pyflakes", Version: "3.2.0", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -316,9 +271,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "tomli", Version: "2.2.1", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, @@ -327,9 +279,6 @@ func TestExtractor_Extract(t *testing.T) { Name: "typing-extensions", Version: "4.12.2", Locations: []string{"testdata/grouped-packages.lock"}, - SourceCode: &extractor.SourceCodeIdentifier{ - Commit: "", - }, Metadata: osv.DepGroupMetadata{ DepGroupVals: []string{}, }, From 1dae38252ae5920e9eefce519a9c1ca87ba15ed8 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 5 Dec 2024 07:33:07 +1300 Subject: [PATCH 09/10] test: introduce `inventory` helper --- .../language/python/uvlock/uvlock_test.go | 139 ++++-------------- 1 file changed, 27 insertions(+), 112 deletions(-) diff --git a/extractor/filesystem/language/python/uvlock/uvlock_test.go b/extractor/filesystem/language/python/uvlock/uvlock_test.go index b4ed1116..26cfea08 100644 --- a/extractor/filesystem/language/python/uvlock/uvlock_test.go +++ b/extractor/filesystem/language/python/uvlock/uvlock_test.go @@ -27,6 +27,19 @@ import ( "github.com/google/osv-scalibr/testing/extracttest" ) +func inventory(t *testing.T, name string, version string, location string) *extractor.Inventory { + t.Helper() + + return &extractor.Inventory{ + Name: name, + Version: version, + Locations: []string{location}, + Metadata: osv.DepGroupMetadata{ + DepGroupVals: []string{}, + }, + } +} + func TestExtractor_FileRequired(t *testing.T) { tests := []struct { name string @@ -112,14 +125,7 @@ func TestExtractor_Extract(t *testing.T) { Path: "testdata/one-package.lock", }, WantInventory: []*extractor.Inventory{ - { - Name: "emoji", - Version: "2.14.0", - Locations: []string{"testdata/one-package.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, + inventory(t, "emoji", "2.14.0", "testdata/one-package.lock"), }, }, { @@ -128,22 +134,8 @@ func TestExtractor_Extract(t *testing.T) { Path: "testdata/two-packages.lock", }, WantInventory: []*extractor.Inventory{ - { - Name: "emoji", - Version: "2.14.0", - Locations: []string{"testdata/two-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, - { - Name: "protobuf", - Version: "4.25.5", - Locations: []string{"testdata/two-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, + inventory(t, "emoji", "2.14.0", "testdata/two-packages.lock"), + inventory(t, "protobuf", "4.25.5", "testdata/two-packages.lock"), }, }, { @@ -171,14 +163,7 @@ func TestExtractor_Extract(t *testing.T) { Path: "testdata/grouped-packages.lock", }, WantInventory: []*extractor.Inventory{ - { - Name: "emoji", - Version: "2.14.0", - Locations: []string{"testdata/grouped-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, + inventory(t, "emoji", "2.14.0", "testdata/grouped-packages.lock"), { Name: "click", Version: "8.1.7", @@ -187,14 +172,7 @@ func TestExtractor_Extract(t *testing.T) { DepGroupVals: []string{"cli"}, }, }, - { - Name: "colorama", - Version: "0.4.6", - Locations: []string{"testdata/grouped-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, + inventory(t, "colorama", "0.4.6", "testdata/grouped-packages.lock"), { Name: "black", Version: "24.10.0", @@ -211,78 +189,15 @@ func TestExtractor_Extract(t *testing.T) { DepGroupVals: []string{"test"}, }, }, - { - Name: "mccabe", - Version: "0.7.0", - Locations: []string{"testdata/grouped-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, - { - Name: "mypy-extensions", - Version: "1.0.0", - Locations: []string{"testdata/grouped-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, - { - Name: "packaging", - Version: "24.2", - Locations: []string{"testdata/grouped-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, - { - Name: "pathspec", - Version: "0.12.1", - Locations: []string{"testdata/grouped-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, - { - Name: "platformdirs", - Version: "4.3.6", - Locations: []string{"testdata/grouped-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, - { - Name: "pycodestyle", - Version: "2.12.1", - Locations: []string{"testdata/grouped-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, - { - Name: "pyflakes", - Version: "3.2.0", - Locations: []string{"testdata/grouped-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, - { - Name: "tomli", - Version: "2.2.1", - Locations: []string{"testdata/grouped-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, - { - Name: "typing-extensions", - Version: "4.12.2", - Locations: []string{"testdata/grouped-packages.lock"}, - Metadata: osv.DepGroupMetadata{ - DepGroupVals: []string{}, - }, - }, + inventory(t, "mccabe", "0.7.0", "testdata/grouped-packages.lock"), + inventory(t, "mypy-extensions", "1.0.0", "testdata/grouped-packages.lock"), + inventory(t, "packaging", "24.2", "testdata/grouped-packages.lock"), + inventory(t, "pathspec", "0.12.1", "testdata/grouped-packages.lock"), + inventory(t, "platformdirs", "4.3.6", "testdata/grouped-packages.lock"), + inventory(t, "pycodestyle", "2.12.1", "testdata/grouped-packages.lock"), + inventory(t, "pyflakes", "3.2.0", "testdata/grouped-packages.lock"), + inventory(t, "tomli", "2.2.1", "testdata/grouped-packages.lock"), + inventory(t, "typing-extensions", "4.12.2", "testdata/grouped-packages.lock"), }, }, } From 029837e7bd954b4b9b56038f594bb0c66bf2f6f4 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 21 Jan 2025 07:11:09 +1300 Subject: [PATCH 10/10] refactor: remove unused `Registry` field --- extractor/filesystem/language/python/uvlock/uvlock.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/extractor/filesystem/language/python/uvlock/uvlock.go b/extractor/filesystem/language/python/uvlock/uvlock.go index 8d69659b..a0aa1caf 100644 --- a/extractor/filesystem/language/python/uvlock/uvlock.go +++ b/extractor/filesystem/language/python/uvlock/uvlock.go @@ -32,9 +32,8 @@ import ( ) type uvLockPackageSource struct { - Virtual string `toml:"virtual"` - Git string `toml:"git"` - Registry string `toml:"registry"` + Virtual string `toml:"virtual"` + Git string `toml:"git"` } type uvLockPackage struct {