Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

requirement, test: Remove preresolved dependency optimization #540

Merged
merged 6 commits into from
Mar 23, 2023
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test: Fix tests and remove obsolete logic
tetsuo-cpp committed Mar 10, 2023
commit 8dc2c0f799984e1cf75171a25163f9cde078409f
58 changes: 1 addition & 57 deletions pip_audit/_dependency_source/requirement.py
Original file line number Diff line number Diff line change
@@ -14,7 +14,6 @@
from typing import IO, Iterator

from packaging.specifiers import SpecifierSet
from packaging.version import Version
from pip_requirements_parser import InstallRequirement, InvalidRequirementLine, RequirementsFile

from pip_audit._dependency_source import (
@@ -25,7 +24,7 @@
)
from pip_audit._fix import ResolvedFixVersion
from pip_audit._service import Dependency
from pip_audit._service.interface import ResolvedDependency, SkippedDependency
from pip_audit._service.interface import ResolvedDependency
from pip_audit._state import AuditState
from pip_audit._virtual_env import VirtualEnv, VirtualEnvError

@@ -203,61 +202,6 @@ def _recover_files(self, tmp_files: list[IO[str]]) -> None:
logger.warning(f"encountered an exception during file recovery: {e}")
continue

def _collect_preresolved_deps(
self, reqs: Iterator[InstallRequirement], require_hashes: bool
) -> Iterator[Dependency]:
"""
Collect pre-resolved (pinned) dependencies.
"""
req_names: set[str] = set()
for req in reqs:
if not req.hash_options and require_hashes:
raise RequirementSourceError(f"requirement {req.dumps()} does not contain a hash")
if req.req is None:
# PEP 508-style URL requirements don't have a pre-declared version, even
# when hashed; the `#egg=name==version` syntax is non-standard and not supported
# by `pip` itself.
#
# In this case, we can't audit the dependency so we should signal to the
# caller that we're skipping it.
yield SkippedDependency(
name=req.requirement_line.line,
skip_reason="could not deduce package version from URL requirement",
)
continue
if self._skip_editable and req.is_editable:
yield SkippedDependency(name=req.name, skip_reason="requirement marked as editable")
if req.marker is not None and not req.marker.evaluate():
continue # pragma: no cover

# This means we have a duplicate requirement for the same package
if req.name in req_names:
raise RequirementSourceError(
f"package {req.name} has duplicate requirements: {str(req)}"
)
req_names.add(req.name)

# NOTE: URL dependencies cannot be pinned, so skipping them
# makes sense (under the same principle of skipping dependencies
# that can't be found on PyPI). This is also consistent with
# what `pip --no-deps` does (installs the URL dependency, but
# not any subdependencies).
if req.is_url:
yield SkippedDependency(
name=req.name,
skip_reason="URL requirements cannot be pinned to a specific package version",
)
elif not req.specifier:
raise RequirementSourceError(f"requirement {req.name} is not pinned: {str(req)}")
else:
pinned_specifier = PINNED_SPECIFIER_RE.match(str(req.specifier))
if pinned_specifier is None:
raise RequirementSourceError(
f"requirement {req.name} is not pinned to an exact version: {str(req)}"
)

yield ResolvedDependency(req.name, Version(pinned_specifier.group("version")))


class RequirementSourceError(DependencySourceError):
"""A requirements-parsing specific `DependencySourceError`."""
174 changes: 63 additions & 111 deletions test/dependency_source/test_requirement.py
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@
requirement,
)
from pip_audit._fix import ResolvedFixVersion
from pip_audit._service import ResolvedDependency, SkippedDependency
from pip_audit._service import ResolvedDependency
from pip_audit._state import AuditState
from pip_audit._virtual_env import VirtualEnvError

@@ -124,6 +124,27 @@ def test_requirement_source_git(req_file):
assert ResolvedDependency(name="uWSGI", version=Version("2.0.20")) in specs


@pytest.mark.online
def test_requirement_source_url(req_file):
source = _init_requirement(
[
(
req_file(),
"https://github.com/pallets/flask/archive/refs/tags/2.0.1.tar.gz\n",
)
],
)

specs = list(source.collect())
assert (
ResolvedDependency(
name="Flask",
version=Version("2.0.1"),
)
in specs
)


@pytest.mark.online
def test_requirement_source_multiple_indexes(req_file):
source = _init_requirement(
@@ -318,7 +339,35 @@ def mock_replace(*_args, **_kwargs):
assert expected_req == f.read().strip()


@pytest.mark.online
def test_requirement_source_require_hashes(req_file):
source = _init_requirement(
[
(
req_file(),
"wheel==0.38.1 "
"--hash=sha256:7a95f9a8dc0924ef318bd55b616112c70903192f524d120acc614f59547a9e1f\n"
"setuptools==67.0.0 "
"--hash=sha256:9d790961ba6219e9ff7d9557622d2fe136816a264dd01d5997cfc057d804853d",
)
],
require_hashes=True,
)

specs = list(source.collect())
assert specs == [
ResolvedDependency(name="wheel", version=Version("0.38.1")),
ResolvedDependency(name="setuptools", version=Version("67.0.0")),
]


@pytest.mark.online
def test_requirement_source_require_hashes_not_fully_resolved(req_file):
# When using `--require-hashes`, `pip` requires a fully resolved list of requirements. If it
# finds a subdependency that is not listed in the requirements file, it will raise an error.
#
# In the case of Flask, this package has lots of subdependencies that aren't listed here so we
# expect an error.
source = _init_requirement(
[
(
@@ -330,12 +379,12 @@ def test_requirement_source_require_hashes(req_file):
require_hashes=True,
)

specs = list(source.collect())
assert specs == [ResolvedDependency("flask", Version("2.0.1"))]
with pytest.raises(DependencySourceError):
list(source.collect())


def test_requirement_source_require_hashes_missing(req_file):
source = _init_requirement([(req_file(), "flask==2.0.1")], require_hashes=True)
source = _init_requirement([(req_file(), "wheel==0.38.1")], require_hashes=True)

# All requirements must be hashed when collecting with `require-hashes`
with pytest.raises(DependencySourceError):
@@ -347,9 +396,9 @@ def test_requirement_source_require_hashes_inferred(req_file):
[
(
req_file(),
"flask==2.0.1 "
"--hash=sha256:a6209ca15eb63fc9385f38e452704113d679511d9574d09b2cf9183ae7d20dc9\n"
"requests==2.0",
"wheel==0.38.1 "
"--hash=sha256:7a95f9a8dc0924ef318bd55b616112c70903192f524d120acc614f59547a9e1f\n"
"setuptools==67.0.0",
)
]
)
@@ -364,10 +413,10 @@ def test_requirement_source_require_hashes_unpinned(req_file):
[
(
req_file(),
"flask==2.0.1 "
"--hash=sha256:a6209ca15eb63fc9385f38e452704113d679511d9574d09b2cf9183ae7d20dc9\n"
"requests>=1.0 "
"--hash=sha256:requests-hash",
"wheel==0.38.1 "
"--hash=sha256:7a95f9a8dc0924ef318bd55b616112c70903192f524d120acc614f59547a9e1f\n"
"setuptools<=67.0.0 "
"--hash=sha256:9d790961ba6219e9ff7d9557622d2fe136816a264dd01d5997cfc057d804853d",
)
]
)
@@ -378,109 +427,12 @@ def test_requirement_source_require_hashes_unpinned(req_file):
list(source.collect())


@pytest.mark.online
def test_requirement_source_no_deps(req_file):
source = _init_requirement([(req_file(), "flask==2.0.1")], no_deps=True)

specs = list(source.collect())
assert specs == [ResolvedDependency("flask", Version("2.0.1"))]


def test_requirement_source_no_deps_unpinned(req_file):
source = _init_requirement([(req_file(), "flask\nrequests==1.0")], no_deps=True)

# When dependency resolution is disabled, all requirements must be pinned.
with pytest.raises(DependencySourceError):
list(source.collect())


def test_requirement_source_no_deps_not_exact_version(req_file):
source = _init_requirement([(req_file(), "flask==1.0\nrequests>=1.0")], no_deps=True)

# When dependency resolution is disabled, all requirements must be pinned.
with pytest.raises(DependencySourceError):
list(source.collect())


def test_requirement_source_no_deps_unpinned_url(req_file):
source = _init_requirement(
[
(
req_file(),
"https://github.com/pallets/flask/archive/refs/tags/2.0.1.tar.gz#egg=flask\n",
)
],
no_deps=True,
)

assert list(source.collect()) == [
SkippedDependency(
name="flask",
skip_reason="URL requirements cannot be pinned to a specific package version",
)
]


def test_requirement_source_no_deps_editable_with_egg_fragment(req_file):
source = _init_requirement([(req_file(), "-e file:flask.py#egg=flask==2.0.1")], no_deps=True)

specs = list(source.collect())
assert (
SkippedDependency(
name="flask",
skip_reason="URL requirements cannot be pinned to a specific package version",
)
in specs
)


def test_requirement_source_no_deps_editable_without_egg_fragment(req_file):
source = _init_requirement([(req_file(), "-e file:flask.py")], no_deps=True)

specs = list(source.collect())
assert (
SkippedDependency(
name="-e file:flask.py",
skip_reason="could not deduce package version from URL requirement",
)
in specs
)


def test_requirement_source_no_deps_non_editable_without_egg_fragment(req_file):
source = _init_requirement(
[
(
req_file(),
"git+https://github.com/unbit/uwsgi.git@1bb9ad77c6d2d310c2d6d1d9ad62de61f725b824",
)
],
no_deps=True,
)

specs = list(source.collect())
assert (
SkippedDependency(
name="git+https://github.com/unbit/uwsgi.git@1bb9ad77c6d2d310c2d6d1d9ad62de61f725b824",
skip_reason="could not deduce package version from URL requirement",
)
in specs
)


def test_requirement_source_no_deps_editable_skip(req_file):
source = _init_requirement(
[(req_file(), "-e file:flask.py#egg=flask==2.0.1")], no_deps=True, skip_editable=True
)

specs = list(source.collect())
assert SkippedDependency(name="flask", skip_reason="requirement marked as editable") in specs


def test_requirement_source_no_deps_duplicate_dependencies(req_file):
source = _init_requirement([(req_file(), "flask==1.0\nflask==1.0")], no_deps=True)

with pytest.raises(DependencySourceError):
list(source.collect())
assert specs == [ResolvedDependency("Flask", Version("2.0.1"))]


def test_requirement_source_fix_explicit_subdep(monkeypatch, req_file):
@@ -517,7 +469,7 @@ def test_requirement_source_fix_explicit_subdep(monkeypatch, req_file):
assert len(logger.warning.calls) == 1


def test_requirement_source_fix_explicit_subdep_multiple_reqs(monkeypatch, req_file):
def test_requirement_source_fix_explicit_subdep_multiple_reqs(req_file):
# Recreate the vulnerable subdependency case.
source = _init_requirement([(req_file(), "flask==2.0.1")])
flask_deps = source.collect()