From f528077d011720bc973045af0d5c9912dae0356e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 13 Aug 2024 05:04:39 -0400 Subject: [PATCH 01/12] Add rudimentary inferred_deps to the discovery module. --- discovery.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/discovery.py b/discovery.py index f2f065d..561d338 100644 --- a/discovery.py +++ b/discovery.py @@ -22,6 +22,9 @@ from packaging.version import Version from pip_run import scripts +from . import imports +from . import pypi + log = logging.getLogger(__name__) mimetypes.add_type('text/plain', '', strict=True) @@ -127,6 +130,17 @@ def read_deps(): return scripts.DepsReader.search(['__init__.py']) +def inferred_deps(): + """ + Infer deps from module imports. + """ + return [ + pypi.distribution_for(imp.relative_to(best_name())) + for module in pathlib.Path().glob('*.py') + for imp in imports.get_module_imports(module) + ] + + def extras_from_dep(dep): try: markers = packaging.requirements.Requirement(dep).marker._markers From 7095cd65d1e8d82b090549c048088d93cb2f2d66 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 13 Aug 2024 05:10:03 -0400 Subject: [PATCH 02/12] Trap errors when unable to resolve distributions. --- discovery.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/discovery.py b/discovery.py index 561d338..4d693b7 100644 --- a/discovery.py +++ b/discovery.py @@ -8,6 +8,7 @@ import operator import pathlib import subprocess +import sys import types import urllib.parse from collections.abc import Mapping @@ -134,11 +135,16 @@ def inferred_deps(): """ Infer deps from module imports. """ - return [ - pypi.distribution_for(imp.relative_to(best_name())) + names = [ + imp.relative_to(best_name()) for module in pathlib.Path().glob('*.py') for imp in imports.get_module_imports(module) ] + for name in names: + try: + yield pypi.distribution_for(name) + except Exception: + print("Error resolving import", name, file=sys.stderr) def extras_from_dep(dep): From 65c2ac0ddb746b9ae356daf15bd270f62f0e9309 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 13 Aug 2024 05:16:04 -0400 Subject: [PATCH 03/12] Only include third-party imports and ignore the local project. --- discovery.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/discovery.py b/discovery.py index 4d693b7..0319056 100644 --- a/discovery.py +++ b/discovery.py @@ -139,6 +139,8 @@ def inferred_deps(): imp.relative_to(best_name()) for module in pathlib.Path().glob('*.py') for imp in imports.get_module_imports(module) + if not imp.builtin() + and not imp.relative_to(best_name()).startswith(best_name()) ] for name in names: try: From 23fbd4013a835c92740c10780f1d51e5a2eeb40d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 19 Aug 2024 06:45:51 -0400 Subject: [PATCH 04/12] Resolve the source files using 'git ls-files'. Now inferred deps resolves dependencies from all sources. --- discovery.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/discovery.py b/discovery.py index 0319056..c832463 100644 --- a/discovery.py +++ b/discovery.py @@ -131,13 +131,30 @@ def read_deps(): return scripts.DepsReader.search(['__init__.py']) +def source_files(): + """ + Return all files in the source distribution. + + >>> list(source_files()) + [...Path('discovery.py')...] + """ + return ( + pathlib.Path(path) + for path in subprocess.check_output(['git', 'ls-files'], text=True).splitlines() + ) + + +def is_python(path: pathlib.Path) -> bool: + return path.suffix == '.py' + + def inferred_deps(): """ Infer deps from module imports. """ names = [ imp.relative_to(best_name()) - for module in pathlib.Path().glob('*.py') + for module in filter(is_python, source_files()) for imp in imports.get_module_imports(module) if not imp.builtin() and not imp.relative_to(best_name()).startswith(best_name()) From 8d5afe827a02303efba06ff4dbfde2229bb529d6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 30 Aug 2024 17:44:00 -0400 Subject: [PATCH 05/12] Add support for generating optional dependencies for tests and docs. --- discovery.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/discovery.py b/discovery.py index c832463..bfe2708 100644 --- a/discovery.py +++ b/discovery.py @@ -153,19 +153,39 @@ def inferred_deps(): Infer deps from module imports. """ names = [ - imp.relative_to(best_name()) + (imp.relative_to(best_name()), module) for module in filter(is_python, source_files()) for imp in imports.get_module_imports(module) if not imp.builtin() and not imp.relative_to(best_name()).startswith(best_name()) ] - for name in names: + for name, module in names: try: - yield pypi.distribution_for(name) + yield pypi.distribution_for(name) + extra_for(module) except Exception: print("Error resolving import", name, file=sys.stderr) +def extra_for(module: pathlib.Path) -> str: + """ + Emit appropriate extra marker if relevant to the module's path. + + >>> extra_for(pathlib.Path('foo/bar')) + '' + >>> extra_for(pathlib.Path('foo.py')) + '' + >>> extra_for(pathlib.Path('tests/foo')) + '; extra=="test"' + >>> extra_for(pathlib.Path('docs/conf.py')) + '; extra=="doc"' + """ + mapping = dict(tests='test', docs='doc') + try: + return f'; extra=="{mapping[str(module.parents[-2])]}"' + except (KeyError, IndexError): + return '' + + def extras_from_dep(dep): try: markers = packaging.requirements.Requirement(dep).marker._markers From 1fdeeb69af5a6400e29dea0c1c7bec687cb486a6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 30 Aug 2024 17:59:07 -0400 Subject: [PATCH 06/12] Add discovery.combined_deps. Gives deference to the declared deps so they can include a more restrictive scope. --- __init__.py | 1 + discovery.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/__init__.py b/__init__.py index 1f5a369..54296a9 100644 --- a/__init__.py +++ b/__init__.py @@ -11,6 +11,7 @@ 'jaraco.compat >= 4.1', 'importlib_resources; python_version < "3.12"', 'jaraco.vcs', + 'more-itertools', ] diff --git a/discovery.py b/discovery.py index bfe2708..1968d5d 100644 --- a/discovery.py +++ b/discovery.py @@ -20,6 +20,7 @@ import setuptools_scm from jaraco.compat.py38 import r_fix from jaraco.context import suppress +from more_itertools import unique_everseen from packaging.version import Version from pip_run import scripts @@ -166,6 +167,16 @@ def inferred_deps(): print("Error resolving import", name, file=sys.stderr) +def combined_deps(): + def package_name(dep): + return packaging.requirements.Requirement(dep).name + + return unique_everseen( + itertools.chain(read_deps(), inferred_deps()), + key=package_name, + ) + + def extra_for(module: pathlib.Path) -> str: """ Emit appropriate extra marker if relevant to the module's path. From 2fd85d74eade1e3256d1470fe1b03eb9d656068c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 30 Aug 2024 18:09:42 -0400 Subject: [PATCH 07/12] Normalize package name when combining. --- discovery.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/discovery.py b/discovery.py index 1968d5d..dc25044 100644 --- a/discovery.py +++ b/discovery.py @@ -7,6 +7,7 @@ import mimetypes import operator import pathlib +import re import subprocess import sys import types @@ -168,8 +169,11 @@ def inferred_deps(): def combined_deps(): + def normalize(name): + return re.sub(r'[.-_]', '-', name).lower() + def package_name(dep): - return packaging.requirements.Requirement(dep).name + return normalize(packaging.requirements.Requirement(dep).name) return unique_everseen( itertools.chain(read_deps(), inferred_deps()), From 1b6e6bc7f8b960edbf68e063bafead7d2c14e16e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 30 Aug 2024 18:36:26 -0400 Subject: [PATCH 08/12] Rename read_deps to declared_deps. --- discovery.py | 4 ++-- metadata.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/discovery.py b/discovery.py index dc25044..e189afb 100644 --- a/discovery.py +++ b/discovery.py @@ -126,7 +126,7 @@ def python_requires_supported(): return f'>= {min_ver}' -def read_deps(): +def declared_deps(): """ Read deps from ``__init__.py``. """ @@ -176,7 +176,7 @@ def package_name(dep): return normalize(packaging.requirements.Requirement(dep).name) return unique_everseen( - itertools.chain(read_deps(), inferred_deps()), + itertools.chain(declared_deps(), inferred_deps()), key=package_name, ) diff --git a/metadata.py b/metadata.py index 8a9b2a4..92607f0 100644 --- a/metadata.py +++ b/metadata.py @@ -81,7 +81,7 @@ def _discover_fields(): yield 'Author-Email', discovery.author_from_vcs() yield 'Summary', discovery.summary_from_github() yield 'Requires-Python', discovery.python_requires_supported() - deps = list(discovery.read_deps()) + deps = list(discovery.declared_deps()) for dep in deps: yield 'Requires-Dist', dep for extra in discovery.full_extras(discovery.extras_from_deps(deps)): From 9492957aad8b94fe62d595181ba1a1bcfbe8bee3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 30 Aug 2024 22:07:12 -0400 Subject: [PATCH 09/12] Rely on coherent.deps for import detection. --- __init__.py | 1 + discovery.py | 4 ++-- metadata.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/__init__.py b/__init__.py index 54296a9..ff442ea 100644 --- a/__init__.py +++ b/__init__.py @@ -12,6 +12,7 @@ 'importlib_resources; python_version < "3.12"', 'jaraco.vcs', 'more-itertools', + 'coherent.deps', ] diff --git a/discovery.py b/discovery.py index e189afb..0b20c35 100644 --- a/discovery.py +++ b/discovery.py @@ -25,8 +25,8 @@ from packaging.version import Version from pip_run import scripts -from . import imports -from . import pypi +from ..deps import imports +from ..deps import pypi log = logging.getLogger(__name__) diff --git a/metadata.py b/metadata.py index 92607f0..9604b76 100644 --- a/metadata.py +++ b/metadata.py @@ -81,7 +81,7 @@ def _discover_fields(): yield 'Author-Email', discovery.author_from_vcs() yield 'Summary', discovery.summary_from_github() yield 'Requires-Python', discovery.python_requires_supported() - deps = list(discovery.declared_deps()) + deps = list(discovery.combined_deps()) for dep in deps: yield 'Requires-Dist', dep for extra in discovery.full_extras(discovery.extras_from_deps(deps)): From 2dd0b6ce32aed46bddeea5d6380494c2e0a009fe Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 30 Aug 2024 22:54:31 -0400 Subject: [PATCH 10/12] Expand the test to ensure that multiple levels are covered. --- discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery.py b/discovery.py index 0b20c35..d44043a 100644 --- a/discovery.py +++ b/discovery.py @@ -189,7 +189,7 @@ def extra_for(module: pathlib.Path) -> str: '' >>> extra_for(pathlib.Path('foo.py')) '' - >>> extra_for(pathlib.Path('tests/foo')) + >>> extra_for(pathlib.Path('tests/functional/foo')) '; extra=="test"' >>> extra_for(pathlib.Path('docs/conf.py')) '; extra=="doc"' From 0481e87baf85f2496e3a79ee8429430c2846c4ab Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 30 Aug 2024 22:55:01 -0400 Subject: [PATCH 11/12] Wrap parents in a list to work around bugs in pathlib on Windows. --- discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discovery.py b/discovery.py index d44043a..05298e1 100644 --- a/discovery.py +++ b/discovery.py @@ -196,7 +196,7 @@ def extra_for(module: pathlib.Path) -> str: """ mapping = dict(tests='test', docs='doc') try: - return f'; extra=="{mapping[str(module.parents[-2])]}"' + return f'; extra=="{mapping[str(list(module.parents)[-2])]}"' except (KeyError, IndexError): return '' From a3a390a1555be840758f5177e93ef3813d1c40c4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 31 Aug 2024 03:49:39 -0400 Subject: [PATCH 12/12] Sort imports --- discovery.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/discovery.py b/discovery.py index 05298e1..c9edb50 100644 --- a/discovery.py +++ b/discovery.py @@ -25,8 +25,7 @@ from packaging.version import Version from pip_run import scripts -from ..deps import imports -from ..deps import pypi +from ..deps import imports, pypi log = logging.getLogger(__name__)