diff --git a/.cruft.json b/.cruft.json index 4c1d0fc..e1ce2ba 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,30 +1,23 @@ { "template": "https://github.com/usnistgov/cookiecutter-nist-python.git", - "commit": "7ba78569cba4c2b39c2706d1f95b74c919b310cf", + "commit": "8fe5103d25de7e7d13d400f599c6653866a69afc", "checkout": "develop", "context": { "cookiecutter": { + "project_name": "pyproject2conda", + "project_slug": "pyproject2conda", + "project_short_description": "A script to convert a Python project declared on a pyproject.toml to a conda environment.", "full_name": "William P. Krekelberg", "email": "wpk@nist.gov", "github_username": "usnistgov", "pypi_username": "conda-forge", "conda_channel": "wpk-nist", - "project_name": "pyproject2conda", - "project_slug": "pyproject2conda", - "_copy_without_render": [ - "*.html", - "docs/_templates/*.rst", - "docs/_templates/autosummary/*.rst", - "docs/_templates/autodocsumm/*.rst", - "docs/_static/css/*", - "docs/_static/js/*", - "changelog.d/templates/*.j2", - "changelog.d/templates/auto-changelog/*.jinja2" - ], - "project_short_description": "A script to convert a Python project declared on a pyproject.toml to a conda environment.", - "command_line_interface": "Typer", - "sphinx_use_autodocsumm": "y", + "command_line_interface": "typer", + "sphinx_use_autodocsumm": true, "sphinx_theme": "sphinx_book_theme", + "year": "2023", + "__answers": "", + "_copy_without_render": [], "_template": "https://github.com/usnistgov/cookiecutter-nist-python.git" } }, diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b38489..f7037ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ default_install_hook_types: repos: # * Top level - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -35,11 +35,11 @@ repos: stages: [commit] additional_dependencies: - prettier-plugin-toml - exclude: ^requirements/lock/.*[.]yml + exclude: ^requirements/lock/.*[.]yml|^.copier-answers.ya?ml # * Markdown - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.9.2 + rev: v0.10.0 hooks: - id: markdownlint-cli2 args: ["--style prettier"] @@ -47,11 +47,11 @@ repos: # * Linting - repo: https://github.com/charliermarsh/ruff-pre-commit # Ruff version. - rev: "v0.0.289" + rev: "v0.1.3" hooks: - id: ruff - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 23.10.1 hooks: # NOTE: nbQA for notebook formatting - id: black @@ -60,19 +60,19 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==23.9.1 + - black==23.10.1 # exclude: ^README.md - repo: https://github.com/nbQA-dev/nbQA rev: 1.7.0 hooks: - id: nbqa-ruff - additional_dependencies: [ruff==0.0.289] + additional_dependencies: [ruff==0.1.3] - id: nbqa-black - additional_dependencies: [black==23.9.1] + additional_dependencies: [black==23.10.1] # * Commit message - repo: https://github.com/commitizen-tools/commitizen - rev: 3.9.0 + rev: 3.12.0 hooks: - id: commitizen stages: [commit-msg] @@ -85,7 +85,7 @@ repos: - id: isort stages: [manual] - repo: https://github.com/asottile/pyupgrade - rev: v3.10.1 + rev: v3.15.0 hooks: - id: pyupgrade stages: [manual] @@ -113,7 +113,7 @@ repos: stages: [manual] # ** spelling - repo: https://github.com/codespell-project/codespell - rev: v2.2.5 + rev: v2.2.6 hooks: - id: codespell types_or: [python, rst, markdown, cython, c] diff --git a/CHANGELOG.md b/CHANGELOG.md index 844cdea..63a0d88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ + + # Changelog @@ -6,11 +8,29 @@ Changelog for `pyproject2conda` ## Unreleased -See the fragment files in -[changelog.d](https://github.com/usnistgov/pyproject2conda) +[changelog.d]: https://github.com/usnistgov/pyproject2conda/tree/main/changelog.d + +See the fragment files in [changelog.d] + + + + +## v0.9.0 — 2023-11-14 + +### Added + +- Default is now to remove whitespace from dependencies. For example, the + dependency `module > 0.1` will become `module>0.1`. To override this + behaviour, pass the option `--no-remove-whitespace`. +- Now supports python version `>3.8,<=3.12` +- Can now specify `extras = false` in pyprojec.toml to skip any extras. The + default (`extras = true`) is the same as `extras = [env_name]` where + `env_name` is the name of the environment (e.g., + `tool.pyproject2conda.envs.env_name`). + ## v0.8.0 — 2023-10-02 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1a4b7f..cc317bd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,9 @@ You can contribute in many ways: ## Types of Contributions + [issues]: https://github.com/usnistgov/pyproject2conda/issues + ### Report Bugs @@ -31,9 +33,9 @@ and "help wanted" is open to whoever wants to implement it. ### Write Documentation -`pyproject2conda` could always use more documentation, whether as part of the -official `pyproject2conda` docs, in docstrings, or even on the web in blog -posts, articles, and such. +This project could always use more documentation, whether as part of the +official docs, in docstrings, or even on the web in blog posts, articles, and +such. ### Submit Feedback @@ -48,10 +50,9 @@ If you are proposing a feature: ## Making a contribution -Ready to contribute? Here's how to set up `pyproject2conda` for local -development. +Ready to contribute? Here's how to make a contribution. -- Fork the `pyproject2conda` repo on GitHub. +- Fork the repo on GitHub. - Clone your fork locally: @@ -278,6 +279,8 @@ where commands can be one of: [ghp-import](https://github.com/c-w/ghp-import)) - livehtml : Live documentation updates - open : open the documentation in a web browser +- serve : Serve the created documentation webpage (Need this to view javescript + in created pages). ## Testing with nox @@ -517,22 +520,23 @@ Before you submit a pull request, check that it meets these guidelines: [setuptools_scm]: https://github.com/pypa/setuptools_scm -Versioning is handled with [setuptools_scm].The package version is set by the +Versioning is handled with [setuptools_scm]. The package version is set by the git tag. For convenience, you can override the version with nox setting `--version ...`. This is useful for updating the docs, etc. -We use the `write_to` option to [setuptools_scm]. This stores the current -version in `_version.py`. Note that if you build the package (or, build docs -with the `--version` flag), this will overwrite information in `_version.py` in -the `src` directory. To refresh the version, run: +Note that the version in a given environment/session can become stale. The +easiest way to update the installed package version version is to reinstall the +package. This can be done using the following: ```bash -make version-scm +pip install -e . --no-deps ``` -Note also that the file `_version.py` SHOULD NOT be tracked by git. It will be -autogenerated when building the package. This scheme avoids having to install -`setuptools-scm` (and `setuptools`) in each environment. +To do this in a given session, use: + +```bash +nox -s {session} -- -P/--update-package +``` ## Serving the documentation @@ -542,4 +546,9 @@ To view to documentation with js headers/footers, you'll need to serve them: python -m http.server -d docs/_build/html ``` -Then open the address `localhost:8000` in a webbrowser. +Then open the address `localhost:8000` in a webbrowser. Alternatively, you can +run: + +```bash +nox -s docs -- -d serve +``` diff --git a/Makefile b/Makefile index 4ff8a3e..20aa023 100644 --- a/Makefile +++ b/Makefile @@ -141,6 +141,9 @@ version-scm: ## check/update version of package with setuptools-scm version-import: ## check version from python import -python -c 'import pyproject2conda; print(pyproject2conda.__version__)' +version-update: ## update version using nox + nox -s update-version-scm + version: version-scm version-import ################################################################################ @@ -149,9 +152,10 @@ version: version-scm version-import .PHONY: requirements requirements: ## rebuild all requirements/environment files nox -s requirements - -requirements/%.yaml: pyproject.toml requirements -requirements/%.txt: pyproject.toml requirements +requirements/%.yaml: pyproject.toml + nox -s requirements +requirements/%.txt: pyproject.toml + nox -s requirements ################################################################################ # * NOX @@ -283,7 +287,8 @@ pytest-nbval: ## run pytest --nbval .PHONY: cog-readme cog-readme: ## apply cog to README.md - P2C_COLUMNS=100 cog -rP README.md + # P2C_COLUMNS=100 cog -rP README.md + nox -s cog pre-commit run markdownlint --files README.md # NOTE: Some special stuff for README.pdf diff --git a/README.md b/README.md index ea30a43..d512ecb 100644 --- a/README.md +++ b/README.md @@ -507,6 +507,18 @@ python = ["3.10"] user_config = "config/userconfig.toml" default_envs = ["test", "dev", "dist-pypi"] +[tool.pyproject2conda.envs.base] +style = ["requirements"] +# Note that the default value for `extras` is the name of the environment. +# To have no extras, either pass +# extras = [] +# or +# +extras = false + +# +# A value of `extras = true` also implies using the environment name +# as the extras. [tool.pyproject2conda.envs."test-extras"] extras = ["test"] style = ["yaml", "requirements"] @@ -610,6 +622,18 @@ python = ["3.10"] user_config = "config/userconfig.toml" default_envs = ["test", "dev", "dist-pypi"] +[tool.pyproject2conda.envs.base] +style = ["requirements"] +# Note that the default value for `extras` is the name of the environment. +# To have no extras, either pass +# extras = [] +# or +# +extras = false + +# +# A value of `extras = true` also implies using the environment name +# as the extras. [tool.pyproject2conda.envs."test-extras"] extras = ["test"] style = ["yaml", "requirements"] @@ -629,11 +653,16 @@ python = ["3.10", "3.11"] run through the command `pyproject2conda project` (or `p2c project`): - + ```bash $ p2c project -f tests/data/test-pyproject.toml --dry # -------------------- +# Creating requirements base.txt +athing +bthing +cthing;python_version<'3.10' +# -------------------- # Creating yaml py310-test-extras.yaml channels: - conda-forge @@ -673,11 +702,6 @@ dependencies: - python=3.11 - bthing-conda - conda-forge::pytest - - pandas - - pip - - pip: - - athing -# -------------------- ... ``` diff --git a/docs/conf.py b/docs/conf.py index 7d6c757..2f5b230 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,7 +58,6 @@ # "sphinx_design" ## myst stuff "myst_nb", - # project specific "sphinx_click", ] @@ -114,7 +113,10 @@ # nb_execution_mode = "auto" # set the kernel name -nb_kernel_rgx_aliases = {"pyproject2conda.*": "python3", "conda.*": "python3"} +nb_kernel_rgx_aliases = { + "pyproject2conda.*": "python3", + "conda.*": "python3", +} nb_execution_allow_errors = True @@ -478,7 +480,9 @@ def linkcode_resolve(domain, info): else: linespec = "" + # fmt: off fn = os.path.relpath(fn, start=os.path.dirname(pyproject2conda.__file__)) + # fmt: on return f"https://github.com/{github_username}/pyproject2conda/blob/{html_context['github_version']}/src/pyproject2conda/{fn}{linespec}" diff --git a/noxfile.py b/noxfile.py index 1cbd226..8cee2bf 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,21 +4,34 @@ import shutil import sys + +# Should only use on python version > 3.10 +assert sys.version_info >= (3, 10) + from dataclasses import replace # noqa from pathlib import Path from typing import ( - Annotated, Any, Callable, Iterator, Sequence, - TypeAlias, TypeVar, cast, ) -import nox -from noxopt import NoxOpt, Option, Session +if sys.version_info > (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + +if sys.version_info > (3, 9): + from typing import Annotated +else: + from typing_extensions import Annotated + + +import nox # type: ignore[unused-ignore,import] +from noxopt import NoxOpt, Option, Session # type: ignore[unused-ignore,import] # fmt: off sys.path.insert(0, ".") @@ -61,7 +74,7 @@ # * Options ---------------------------------------------------------------------------- -PYTHON_ALL_VERSIONS = ["3.8", "3.9", "3.10", "3.11"] +PYTHON_ALL_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"] PYTHON_DEFAULT_VERSION = "3.10" # conda/mamba @@ -88,6 +101,9 @@ DEFAULT_SESSION_VENV = cast(C[F], group.session(python=PYTHON_DEFAULT_VERSION)) # type: ignore ALL_SESSION_VENV = cast(C[F], group.session(python=PYTHON_ALL_VERSIONS)) # type: ignore +NOPYTHON_SESSION = cast(C[F], group.session(python=False)) # type: ignore +INHERITED_SESSION_VENV = cast(C[F], group.session) # type: ignore + OPTS_OPT = Option(nargs="*", type=str) # SET_KERNEL_OPT = Option(type=bool, help="If True, try to set the kernel name") RUN_OPT = Option( @@ -101,20 +117,20 @@ LOCK_OPT = Option(type=bool, help="If True, use conda-lock") -def opts_annotated(**kwargs: Any): # type: ignore - return Annotated[list[str], replace(OPTS_OPT, **kwargs)] +def opts_annotated(**kwargs: Any): # type: ignore[unused-ignore,no-untyped-def] + return Annotated["list[str]", replace(OPTS_OPT, **kwargs)] -def cmd_annotated(**kwargs: Any): # type: ignore - return Annotated[list[str], replace(CMD_OPT, **kwargs)] +def cmd_annotated(**kwargs: Any): # type: ignore[unused-ignore,no-untyped-def] + return Annotated["list[str]", replace(CMD_OPT, **kwargs)] -def run_annotated(**kwargs: Any): # type: ignore - return Annotated[list[list[str]], replace(RUN_OPT, **kwargs)] +def run_annotated(**kwargs: Any): # type: ignore[unused-ignore,no-untyped-def] + return Annotated["list[list[str]]", replace(RUN_OPT, **kwargs)] LOCK_CLI = Annotated[bool, LOCK_OPT] -RUN_CLI = Annotated[list[list[str]], RUN_OPT] +RUN_CLI = Annotated["list[list[str]]", RUN_OPT] TEST_OPTS_CLI = opts_annotated(help="extra arguments/flags to pytest") DEV_EXTRAS_CLI = cmd_annotated(help="extras included in user dev environment") PYTHON_PATHS_CLI = cmd_annotated(help="python paths to append to PATHS") @@ -128,6 +144,16 @@ def run_annotated(**kwargs: Any): # type: ignore ), ] + +UPDATE_PACKAGE_CLI = Annotated[ + bool, + Option( + type=bool, + help="If True, and session uses package, reinstall package", + flags=("--update-package", "-P"), + ), +] + VERSION_CLI = Annotated[ str, Option(type=str, help="Version to substitute or check against") ] @@ -149,6 +175,7 @@ def dev( dev_run: RUN_CLI = [], # noqa lock: LOCK_CLI = False, update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, log_session: bool = False, ) -> None: """Create dev env using conda.""" @@ -161,6 +188,7 @@ def dev( display_name=f"{PACKAGE_NAME}-dev", install_package=True, update=update, + update_package=update_package, log_session=log_session, ) session_run_commands(session, dev_run) @@ -173,6 +201,7 @@ def dev_venv( dev_run: RUN_CLI = [], # noqa lock: LOCK_CLI = False, update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, log_session: bool = False, ) -> None: """Create dev env using virtualenv.""" @@ -186,14 +215,15 @@ def dev_venv( display_name=f"{PACKAGE_NAME}-dev-venv", install_package=True, update=update, + update_package=update_package, log_session=log_session, ) session_run_commands(session, dev_run) # ** bootstrap -@group.session(python=False) # type: ignore -def bootstrap(session: Session): +@NOPYTHON_SESSION +def bootstrap(session: Session) -> None: """Run config, reqs, and dev""" session.notify("config") @@ -202,7 +232,7 @@ def bootstrap(session: Session): # ** config -@group.session(python=False) # type: ignore +@NOPYTHON_SESSION def config( session: Session, dev_extras: DEV_EXTRAS_CLI = [], # type: ignore # noqa @@ -220,7 +250,7 @@ def config( # ** requirements -@group.session(python=False) # type: ignore +@NOPYTHON_SESSION def pyproject2conda( session: Session, update: UPDATE_CLI = False, @@ -229,7 +259,7 @@ def pyproject2conda( session.notify("requirements") -@group.session +@INHERITED_SESSION_VENV def requirements( session: Session, update: UPDATE_CLI = False, @@ -359,6 +389,7 @@ def test( test_run: RUN_CLI = [], # noqa lock: LOCK_CLI = False, update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, log_session: bool = False, no_cov: bool = False, ) -> None: @@ -370,6 +401,7 @@ def test( lock=lock, install_package=True, update=update, + update_package=update_package, log_session=log_session, ) @@ -390,6 +422,7 @@ def test_venv( test_run: RUN_CLI = [], # noqa lock: LOCK_CLI = False, # pyright: ignore update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, log_session: bool = False, no_cov: bool = False, ) -> None: @@ -401,6 +434,7 @@ def test_venv( install_package=True, requirement_paths="test.txt", update=update, + update_package=update_package, log_session=log_session, ) @@ -423,7 +457,7 @@ def _coverage( session_run_commands(session, run) if not cmd and not run and not run_internal: - cmd = ["combine", "report"] + cmd = ["combine", "html"] session.log(f"{cmd}") @@ -442,7 +476,7 @@ def _coverage( session_run_commands(session, run_internal, external=False) -@DEFAULT_SESSION_VENV +@INHERITED_SESSION_VENV def coverage( session: Session, coverage_cmd: cmd_annotated( # type: ignore @@ -488,6 +522,10 @@ def _docs( if open_page := "open" in cmd: cmd.remove("open") + if serve := "serve" in cmd: + open_webpage(url="http://localhost:8000") + cmd.remove("serve") + if cmd: args = ["make", "-C", "docs"] + combine_list_str(cmd) session.run(*args, external=True) @@ -495,6 +533,18 @@ def _docs( if open_page: open_webpage(path="./docs/_build/html/index.html") + if serve and "livehtml" not in cmd: + session.run( + "python", + "-m", + "http.server", + "-d", + "docs/_build/html", + "-b", + "127.0.0.1", + "8000", + ) + @DEFAULT_SESSION def docs( @@ -511,12 +561,14 @@ def docs( "showlinks", "release", "open", + "serve", ], flags=("--docs-cmd", "-d"), ) = (), docs_run: RUN_CLI = [], # noqa lock: LOCK_CLI = False, update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, version: VERSION_CLI = "", log_session: bool = False, ) -> None: @@ -528,6 +580,7 @@ def docs( display_name=f"{PACKAGE_NAME}-docs", install_package=True, update=update, + update_package=update_package, log_session=log_session, ) @@ -551,12 +604,14 @@ def docs_venv( "showlinks", "release", "open", + "serve", ], flags=("--docs-cmd", "-d"), ) = (), docs_run: RUN_CLI = [], # noqa lock: LOCK_CLI = False, update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, version: VERSION_CLI = "", log_session: bool = False, ) -> None: @@ -568,6 +623,7 @@ def docs_venv( display_name=f"{PACKAGE_NAME}-docs-venv", install_package=True, update=update, + update_package=update_package, log_session=log_session, requirement_paths="docs.txt", ) @@ -755,13 +811,13 @@ def dist_conda( elif command == "recipe-cat-full": import tempfile - with tempfile.TemporaryDirectory() as d: + with tempfile.TemporaryDirectory() as d: # type: ignore[assignment,unused-ignore] session.run( "grayskull", "pypi", sdist_path, "-o", - d, + str(d), ) session.run( "cat", str(Path(d) / PACKAGE_NAME / "meta.yaml"), external=True @@ -778,7 +834,7 @@ def dist_conda( # ** lint -@group.session +@INHERITED_SESSION_VENV def lint( session: nox.Session, lint_run: RUN_CLI = [], # noqa @@ -828,6 +884,15 @@ def _run_info(cmd: str) -> None: session.run("which", cmd, external=True) session.run(cmd, "--version", external=True) + if "clean" in cmd: + cmd = [x for x in cmd if x != "clean"] + + for name in [".mypy_cache", ".pytype"]: + p = Path(session.create_tmp()) / name + if p.exists(): + session.log(f"removing cache {p}") + shutil.rmtree(str(p)) + for c in cmd: if not c.startswith("nbqa"): _run_info(c) @@ -849,6 +914,7 @@ def typing( session: nox.Session, typing_cmd: cmd_annotated( # type: ignore choices=[ + "clean", "mypy", "pyright", "pytype", @@ -891,6 +957,7 @@ def typing_venv( session: nox.Session, typing_cmd: cmd_annotated( # type: ignore choices=[ + "clean", "mypy", "pyright", "pytype", @@ -1046,6 +1113,29 @@ def testdist_pypi_condaenv( ) +# * Project specific sessions +@INHERITED_SESSION_VENV +def cog( + session: nox.Session, + update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, + log_session: bool = False, +) -> None: + """Run cog.""" + + pkg_install_venv( + session=session, + name="cog", + requirement_paths="cog.txt", + install_package=True, + update=update, + update_package=update_package, + log_session=log_session, + ) + + session.run("cog", "-rP", "README.md", env={"COLUMNS": "100"}) + + # * Utilities -------------------------------------------------------------------------- def _create_doc_examples_symlinks(session: nox.Session, clean: bool = True) -> None: """Create symlinks from docs/examples/*.md files to /examples/usage/...""" diff --git a/pyproject.toml b/pyproject.toml index 69bbfb1..1dce79b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=61.2", "setuptools_scm[toml]>=7.0"] +requires = ["setuptools>=61.2", "setuptools_scm[toml]>=8.0"] build-backend = "setuptools.build_meta" [project] @@ -20,6 +20,8 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering", ] dynamic = ["readme", "version"] @@ -29,10 +31,9 @@ dependencies = [ "ruamel.yaml", "tomlkit", "typer", - # "rich-click", "packaging", "typing-extensions; python_version<'3.9'", -] # additional packages +] [project.urls] homepage = "https://github.com/usnistgov/pyproject2conda" @@ -50,12 +51,11 @@ test = [ "pytest-sugar", ] dev-extras = [ - "setuptools-scm", # + "setuptools-scm>=8.0", # "pytest-accept", # p2c: -p + # "nbval", "ipython", "ipykernel", - # "nbval", - ] typing-extras = [ "pytype; python_version < '3.11'", # @@ -99,7 +99,6 @@ docs = [ "myst-nb", "sphinx-book-theme", "autodocsumm", - # project specific "sphinx-click", ] # to be parsed with pyproject2conda with --no-base option @@ -111,6 +110,7 @@ dist-conda = [ "conda-verify", "boa", ] +cog = ["cogapp"] [tool.pyproject2conda] user_config = "config/userconfig.toml" @@ -136,13 +136,16 @@ extras = ["test"] [tool.pyproject2conda.envs.dev-base] extras = ["dev"] +[tool.pyproject2conda.envs.cog] +style = "requirements" + [[tool.pyproject2conda.overrides]] envs = ["test-extras", "dist-conda", "dist-pypi"] base = false [[tool.pyproject2conda.overrides]] envs = ["test", "typing", "test-extras"] -python = ["3.8", "3.9", "3.10", "3.11"] +python = ["3.8", "3.9", "3.10", "3.11", "3.12"] [[tool.pyproject2conda.overrides]] envs = ["dist-conda"] @@ -173,7 +176,6 @@ readme = { file = [ [tool.setuptools_scm] fallback_version = "999" -write_to = "src/pyproject2conda/_version.py" [tool.aliases] test = "pytest" @@ -288,7 +290,7 @@ ignore = [ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" [tool.ruff.per-file-ignores] -"src/pyproject2conda.py" = ["UP006", "UP007"] +"src/pyproject2conda/cli.py" = ["UP006", "UP007"] [tool.ruff.isort] known-first-party = ["pyproject2conda"] @@ -343,7 +345,7 @@ use_shortcuts = true [tool.cruft] [tool.mypy] -files = ["src/pyproject2conda", "tests"] +files = ["src", "tests", "noxfile.py", "tools"] show_error_codes = true warn_unused_ignores = true warn_return_any = true @@ -362,16 +364,8 @@ module = [] [tool.pyright] include = ["src", "tests"] -exclude = [ - "**/__pycache__", - ".tox/**", - ".nox/**", - "**/.mypy_cache", - "**/_version.py" -] -# extraPaths = ["."] -# TODO: add strict to pyright -# strict = ["src"] +exclude = ["**/__pycache__", ".tox/**", ".nox/**", "**/.mypy_cache"] +strict = ["src/**/*.py"] pythonVersion = "3.10" typeCheckingMode = "basic" # enable subset of "strict" diff --git a/requirements/cog.txt b/requirements/cog.txt new file mode 100644 index 0000000..afdd21d --- /dev/null +++ b/requirements/cog.txt @@ -0,0 +1,16 @@ +# +# This file is autogenerated by pyproject2conda +# with the following command: +# +# $ pyproject2conda project --verbose +# +# You should not manually edit this file. +# Instead edit the corresponding pyproject.toml file. +# +cogapp +packaging +ruamel.yaml +tomli +tomlkit +typer +typing-extensions;python_version<'3.9' diff --git a/requirements/dev-base.txt b/requirements/dev-base.txt index 2f29a5f..c76fb3b 100644 --- a/requirements/dev-base.txt +++ b/requirements/dev-base.txt @@ -9,18 +9,18 @@ # ipykernel ipython -mypy >= 1.4.1 +mypy>=1.4.1 packaging pytest pytest-accept pytest-cov pytest-sugar pytest-xdist -pytype; python_version < '3.11' +pytype;python_version<'3.11' ruamel.yaml -setuptools-scm +setuptools-scm>=8.0 tomli tomlkit typer types-click -typing-extensions; python_version<'3.9' +typing-extensions;python_version<'3.9' diff --git a/requirements/dev-complete.txt b/requirements/dev-complete.txt index ad0d5f7..99322cf 100644 --- a/requirements/dev-complete.txt +++ b/requirements/dev-complete.txt @@ -9,7 +9,7 @@ # ipykernel ipython -mypy >= 1.4.1 +mypy>=1.4.1 nbqa nox noxopt @@ -21,12 +21,12 @@ pytest-accept pytest-cov pytest-sugar pytest-xdist -pytype; python_version < '3.11' +pytype;python_version<'3.11' ruamel.yaml scriv -setuptools-scm +setuptools-scm>=8.0 tomli tomlkit typer types-click -typing-extensions; python_version<'3.9' +typing-extensions;python_version<'3.9' diff --git a/requirements/dev.txt b/requirements/dev.txt index 03276c3..98070d8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -9,7 +9,7 @@ # ipykernel ipython -mypy >= 1.4.1 +mypy>=1.4.1 nox noxopt packaging @@ -18,11 +18,11 @@ pytest-accept pytest-cov pytest-sugar pytest-xdist -pytype; python_version < '3.11' +pytype;python_version<'3.11' ruamel.yaml -setuptools-scm +setuptools-scm>=8.0 tomli tomlkit typer types-click -typing-extensions; python_version<'3.9' +typing-extensions;python_version<'3.9' diff --git a/requirements/docs.txt b/requirements/docs.txt index 2eca0b3..e99473a 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -14,13 +14,13 @@ myst-nb packaging pyenchant ruamel.yaml -sphinx >= 5.3.0 sphinx-autobuild sphinx-book-theme sphinx-click sphinx-copybutton +sphinx>=5.3.0 sphinxcontrib-spelling tomli tomlkit typer -typing-extensions; python_version<'3.9' +typing-extensions;python_version<'3.9' diff --git a/requirements/py310-dev-base.yaml b/requirements/py310-dev-base.yaml index 54f9d7d..fd92c2a 100644 --- a/requirements/py310-dev-base.yaml +++ b/requirements/py310-dev-base.yaml @@ -13,7 +13,7 @@ dependencies: - python=3.10 - ipykernel - ipython - - mypy >= 1.4.1 + - mypy>=1.4.1 - packaging - pytest - pytest-cov @@ -21,7 +21,7 @@ dependencies: - pytest-xdist - pytype - ruamel.yaml - - setuptools-scm + - setuptools-scm>=8.0 - tomli - tomlkit - typer diff --git a/requirements/py310-dev-complete.yaml b/requirements/py310-dev-complete.yaml index fc3f256..a46a26d 100644 --- a/requirements/py310-dev-complete.yaml +++ b/requirements/py310-dev-complete.yaml @@ -13,7 +13,7 @@ dependencies: - python=3.10 - ipykernel - ipython - - mypy >= 1.4.1 + - mypy>=1.4.1 - nbqa - nox - packaging @@ -25,7 +25,7 @@ dependencies: - pytest-xdist - pytype - ruamel.yaml - - setuptools-scm + - setuptools-scm>=8.0 - tomli - tomlkit - typer diff --git a/requirements/py310-docs.yaml b/requirements/py310-docs.yaml index 40980dc..bda7cfb 100644 --- a/requirements/py310-docs.yaml +++ b/requirements/py310-docs.yaml @@ -18,7 +18,7 @@ dependencies: - packaging - pyenchant - ruamel.yaml - - sphinx >= 5.3.0 + - sphinx>=5.3.0 - sphinx-autobuild - sphinx-book-theme - sphinx-click diff --git a/requirements/py310-typing.yaml b/requirements/py310-typing.yaml index 32f99fb..b286b19 100644 --- a/requirements/py310-typing.yaml +++ b/requirements/py310-typing.yaml @@ -11,7 +11,7 @@ channels: - conda-forge dependencies: - python=3.10 - - mypy >= 1.4.1 + - mypy>=1.4.1 - packaging - pytest - pytest-cov diff --git a/requirements/py311-typing.yaml b/requirements/py311-typing.yaml index 51308cc..6d6d0c1 100644 --- a/requirements/py311-typing.yaml +++ b/requirements/py311-typing.yaml @@ -11,7 +11,7 @@ channels: - conda-forge dependencies: - python=3.11 - - mypy >= 1.4.1 + - mypy>=1.4.1 - packaging - pytest - pytest-cov diff --git a/requirements/py312-test-extras.yaml b/requirements/py312-test-extras.yaml new file mode 100644 index 0000000..8b891d3 --- /dev/null +++ b/requirements/py312-test-extras.yaml @@ -0,0 +1,17 @@ +# +# This file is autogenerated by pyproject2conda +# with the following command: +# +# $ pyproject2conda project --verbose +# +# You should not manually edit this file. +# Instead edit the corresponding pyproject.toml file. +# +channels: + - conda-forge +dependencies: + - python=3.12 + - pytest + - pytest-cov + - pytest-sugar + - pytest-xdist diff --git a/requirements/py312-test.yaml b/requirements/py312-test.yaml new file mode 100644 index 0000000..ace8f0c --- /dev/null +++ b/requirements/py312-test.yaml @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pyproject2conda +# with the following command: +# +# $ pyproject2conda project --verbose +# +# You should not manually edit this file. +# Instead edit the corresponding pyproject.toml file. +# +channels: + - conda-forge +dependencies: + - python=3.12 + - packaging + - pytest + - pytest-cov + - pytest-sugar + - pytest-xdist + - ruamel.yaml + - tomli + - tomlkit + - typer diff --git a/requirements/py312-typing.yaml b/requirements/py312-typing.yaml new file mode 100644 index 0000000..c6e3382 --- /dev/null +++ b/requirements/py312-typing.yaml @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pyproject2conda +# with the following command: +# +# $ pyproject2conda project --verbose +# +# You should not manually edit this file. +# Instead edit the corresponding pyproject.toml file. +# +channels: + - conda-forge +dependencies: + - python=3.12 + - mypy>=1.4.1 + - packaging + - pytest + - pytest-cov + - pytest-sugar + - pytest-xdist + - ruamel.yaml + - tomli + - tomlkit + - typer + - types-click diff --git a/requirements/py38-typing.yaml b/requirements/py38-typing.yaml index ccc2ec7..21b7c3c 100644 --- a/requirements/py38-typing.yaml +++ b/requirements/py38-typing.yaml @@ -11,7 +11,7 @@ channels: - conda-forge dependencies: - python=3.8 - - mypy >= 1.4.1 + - mypy>=1.4.1 - packaging - pytest - pytest-cov diff --git a/requirements/py39-typing.yaml b/requirements/py39-typing.yaml index 6a5938b..03d436c 100644 --- a/requirements/py39-typing.yaml +++ b/requirements/py39-typing.yaml @@ -11,7 +11,7 @@ channels: - conda-forge dependencies: - python=3.9 - - mypy >= 1.4.1 + - mypy>=1.4.1 - packaging - pytest - pytest-cov diff --git a/requirements/test.txt b/requirements/test.txt index 543fe03..306399d 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -16,4 +16,4 @@ ruamel.yaml tomli tomlkit typer -typing-extensions; python_version<'3.9' +typing-extensions;python_version<'3.9' diff --git a/requirements/typing.txt b/requirements/typing.txt index a6d0985..47af58d 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -7,16 +7,16 @@ # You should not manually edit this file. # Instead edit the corresponding pyproject.toml file. # -mypy >= 1.4.1 +mypy>=1.4.1 packaging pytest pytest-cov pytest-sugar pytest-xdist -pytype; python_version < '3.11' +pytype;python_version<'3.11' ruamel.yaml tomli tomlkit typer types-click -typing-extensions; python_version<'3.9' +typing-extensions;python_version<'3.9' diff --git a/src/pyproject2conda/__init__.py b/src/pyproject2conda/__init__.py index 3f67f00..2f12e24 100644 --- a/src/pyproject2conda/__init__.py +++ b/src/pyproject2conda/__init__.py @@ -2,15 +2,15 @@ Top level API (:mod:`pyproject2conda`) ====================================== """ +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _version -from .parser import PyProject2Conda - -# updated versioning scheme try: - from ._version import __version__ -except Exception: # pragma: no cover + __version__ = _version("pyproject2conda") +except PackageNotFoundError: # pragma: no cover __version__ = "999" +from .parser import PyProject2Conda __author__ = """William P. Krekelberg""" __email__ = "wpk@nist.gov" diff --git a/src/pyproject2conda/_typing.py b/src/pyproject2conda/_typing.py index a096bc5..8fce015 100644 --- a/src/pyproject2conda/_typing.py +++ b/src/pyproject2conda/_typing.py @@ -1,6 +1,6 @@ from typing import Any, Callable, TypeVar -from typing_extensions import TypeAlias +from ._typing_compat import TypeAlias FuncType: TypeAlias = Callable[..., Any] diff --git a/src/pyproject2conda/_typing_compat.py b/src/pyproject2conda/_typing_compat.py new file mode 100644 index 0000000..7425efb --- /dev/null +++ b/src/pyproject2conda/_typing_compat.py @@ -0,0 +1,25 @@ +import sys + +if sys.version_info < (3, 10): + from typing_extensions import TypeAlias +else: + from typing import TypeAlias + + +if sys.version_info < (3, 9): + from typing_extensions import Annotated +else: + from typing import Annotated + + +if sys.version_info < (3, 11): + from typing_extensions import Self +else: + from typing import Self + + +__all__ = [ + "TypeAlias", + "Annotated", + "Self", +] diff --git a/src/pyproject2conda/cli.py b/src/pyproject2conda/cli.py index 1dd5d1e..4a0e093 100644 --- a/src/pyproject2conda/cli.py +++ b/src/pyproject2conda/cli.py @@ -1,3 +1,4 @@ +# pyright: reportUnknownMemberType=false """ Console script for pyproject2conda (:mod:`~pyproject2conda.cli`) ================================================================ @@ -6,18 +7,12 @@ import logging import os -import sys from enum import Enum from functools import lru_cache, wraps from inspect import signature from pathlib import Path from typing import Any, Callable, Iterable, List, Optional, Union, cast -if sys.version_info[:2] < (3, 9): - from typing_extensions import Annotated -else: - from typing import Annotated - # from click import click.Context import click import typer @@ -31,6 +26,7 @@ ) from ._typing import R +from ._typing_compat import Annotated # * Logger ----------------------------------------------------------------------------- @@ -108,7 +104,7 @@ def main( PYPROJECT_CLI = Annotated[ Path, - typer.Option( + typer.Option( # pyright: ignore[reportUnknownMemberType] "--file", "-f", help="input pyproject.toml file", @@ -116,23 +112,29 @@ def main( ] EXTRAS_CLI = Annotated[ Optional[List[str]], - typer.Option( + typer.Option( # pyright: ignore[reportUnknownMemberType] "--extra", "-e", - help="Extra dependencies. Can specify multiple times for multiple extras.", + help=""" + Extra dependencies. Can specify multiple times for multiple extras. + Use name `extras` for specifying in `pyproject.toml` + Note thate for `project` application, this parameter defaults to the + name of the environment. If you want no extras, you must pass + `extras = false`. + """, ), ] CHANNEL_CLI = Annotated[ Optional[List[str]], - typer.Option( + typer.Option( # pyright: ignore[reportUnknownMemberType] "--channel", "-c", - help="conda channel. Can specify. Overrides [tool.pyproject2conda.channels]", + help="Conda channel. Can specify. Overrides [tool.pyproject2conda.channels]", ), ] NAME_CLI = Annotated[ Optional[str], - typer.Option( + typer.Option( # pyright: ignore[reportUnknownMemberType] "--name", "-n", help="Name of conda env", @@ -140,7 +142,7 @@ def main( ] OUTPUT_CLI = Annotated[ Optional[Path], - typer.Option( + typer.Option( # pyright: ignore[reportUnknownMemberType] "--output", "-o", help="File to output results", @@ -334,6 +336,17 @@ class Overwrite(str, Enum): ) +REMOVE_WHITESPACE_OPTION = typer.Option( + "--remove-whitespace/--no-remove-whitespace", + help=""" + What to do with whitespace in a dependency. The default (`--remove-whitespace`) is + to remove whitespace in a given dependency. For example, the dependency + `package >= 1.0` will be converted to `package>=1.0`. Pass `--no-remove-whitespace` + to keep the the whitespace in the output. + """, +) + + # * Utils ------------------------------------------------------------------------------ def _get_header_cmd( header: Optional[bool], output: Union[str, Path, None] @@ -406,7 +419,7 @@ def wrapped(*args: Any, **kwargs: Any) -> R: level = logging.WARN elif verbosity == 1: level = logging.INFO - elif verbosity >= 2: # pragma: no cover + else: # pragma: no cover level = logging.DEBUG logger.setLevel(level) @@ -467,6 +480,7 @@ def yaml( deps: DEPS_CLI = None, reqs: REQS_CLI = None, allow_empty: Annotated[bool, ALLOW_EMPTY_OPTION] = False, + remove_whitespace: Annotated[bool, REMOVE_WHITESPACE_OPTION] = True, ) -> None: """Create yaml file from dependencies and optional-dependencies.""" @@ -500,6 +514,7 @@ def yaml( deps=deps, reqs=reqs, allow_empty=allow_empty, + remove_whitespace=remove_whitespace, ) if not output: print(s, end="") @@ -520,6 +535,7 @@ def requirements( verbose: VERBOSE_CLI = None, # pyright: ignore reqs: REQS_CLI = None, allow_empty: Annotated[bool, ALLOW_EMPTY_OPTION] = False, + remove_whitespace: Annotated[bool, REMOVE_WHITESPACE_OPTION] = True, ) -> None: """Create requirements.txt for pip dependencies.""" @@ -539,6 +555,7 @@ def requirements( sort=sort, reqs=reqs, allow_empty=allow_empty, + remove_whitespace=remove_whitespace, ) if not output: print(s, end="") @@ -562,6 +579,7 @@ def project( dry: DRY_CLI = False, user_config: USER_CONFIG_CLI = "infer", allow_empty: Annotated[Optional[bool], ALLOW_EMPTY_OPTION] = None, + remove_whitespace: Annotated[Optional[bool], REMOVE_WHITESPACE_OPTION] = None, ) -> None: """Create multiple environment files from `pyproject.toml` specification.""" from pyproject2conda.config import Config @@ -580,6 +598,7 @@ def project( overwrite=overwrite.value, verbose=verbose, allow_empty=allow_empty, + remove_whitespace=remove_whitespace, ): if dry: # small header diff --git a/src/pyproject2conda/config.py b/src/pyproject2conda/config.py index 71af17a..3acbfc4 100644 --- a/src/pyproject2conda/config.py +++ b/src/pyproject2conda/config.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: from typing import Any, Iterator, Sequence - from typing_extensions import Self + from ._typing_compat import Self # * Utilities @@ -98,7 +98,7 @@ def _get_value( if not isinstance(value, list): value = [value] - return value + return value # pyright: ignore def channels( self, env_name: str | None = None, inherit: bool = True @@ -125,15 +125,33 @@ def python( ) def extras(self, env_name: str) -> list[str]: - """Extras getter""" - return self._get_value( # type: ignore + """ + Extras getter + + * If value is `True` (default), then return [env_name] + * If value is `False`, return [] + * else return list of extras + """ + + val = self._get_value( key="extras", env_name=env_name, inherit=False, - as_list=True, - default=env_name or [], + # as_list=True, + default=env_name, ) + if isinstance(val, bool): + if val: + return [env_name] + else: + return [] + + elif not isinstance(val, list): + val = [val] + + return val # type: ignore + def output(self, env_name: str | None = None) -> str | None: """Output getter""" return self._get_value(key="output", env_name=env_name, inherit=False) # type: ignore @@ -222,6 +240,15 @@ def allow_empty(self, env_name: str | None = None, default: bool = False) -> boo key="allow_empty", env_name=env_name, default=default ) + def remove_whitespace( + self, env_name: str | None = None, default: bool = True + ) -> bool: + return self._get_value( # type: ignore + key="remove_whitespace", + env_name=env_name, + default=default, + ) + def assign_user_config(self, user: Self) -> Self: """Assign user_config to self.""" from copy import deepcopy @@ -241,9 +268,11 @@ def assign_user_config(self, user: Self) -> Self: if u is not None: d = data[key] if isinstance(d, list): - d.extend(u) + assert isinstance(u, list) + d.extend(u) # pyright: ignore elif isinstance(d, dict): - d.update(**u) + assert isinstance(u, dict) + d.update(**u) # pyright: ignore return type(self)(data) @@ -275,6 +304,7 @@ def _iter_yaml( "name", "channels", "allow_empty", + "remove_whitespace", ] data = {k: defaults.get(k, getattr(self, k)(env_name)) for k in keys} @@ -317,6 +347,7 @@ def _iter_reqs( "verbose", "reqs", "allow_empty", + "remove_whitespace", ] output, template, _ = self._get_output_and_templates(env_name, **defaults) diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py index 50b035d..beba1e1 100644 --- a/src/pyproject2conda/parser.py +++ b/src/pyproject2conda/parser.py @@ -1,3 +1,4 @@ +# pyright: reportUnknownMemberType=false, reportGeneralTypeIssues=false """ Parse `pyproject.toml` (:mod:`~pyproject2conda.parser`) ======================================================= @@ -25,7 +26,10 @@ ) if TYPE_CHECKING: - from typing_extensions import Self + import tomlkit.items + import tomlkit.toml_document + + from ._typing_compat import Self import tomlkit from packaging.specifiers import SpecifierSet @@ -53,6 +57,13 @@ def _check_allow_empty(allow_empty: bool) -> str: raise ValueError(msg) +_WHITE_SPACE_REGEX = re.compile(r"\s+") + + +def _remove_whitespace(s: str) -> str: + return re.sub(_WHITE_SPACE_REGEX, "", s) + + # --- Default parser ------------------------------------------------------------------- @@ -203,9 +214,7 @@ def _pyproject_to_value_comment_pairs( unique: bool = True, include_base_dependencies: bool = True, ) -> list[tuple[Tstr_opt, Tstr_opt]]: - package_name = cast( - str, get_in(["project", "name"], data, factory=_factory_empty_tomlkit_Array) - ) + package_name = cast("str | None", get_in(["project", "name"], data, default=None)) if package_name is None: raise ValueError("Must specify `project.package_name` in pyproject.toml") @@ -271,7 +280,7 @@ def _limit_deps_by_python_version( r"(?P.*?);\s*python_version\s*(?P[<=>~]*)\s*[\'|\"](?P.*?)[\'|\"]" ) - output = [] + output: list[str] = [] for dep in deps: if match := matcher.match(dep): if not version or version in SpecifierSet( @@ -338,6 +347,7 @@ def pyproject_to_conda_lists( sort: bool = True, deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, + remove_whitespace: bool = True, ) -> dict[str, Any]: if python_include == "infer": # safer get @@ -376,6 +386,9 @@ def pyproject_to_conda_lists( output["dependencies"], python_version ) + if remove_whitespace: + output = {k: [_remove_whitespace(x) for x in v] for k, v in output.items()} + return output @@ -393,6 +406,7 @@ def pyproject_to_conda( deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, allow_empty: bool = False, + remove_whitespace: bool = True, ) -> str: output = pyproject_to_conda_lists( data=data, @@ -404,6 +418,7 @@ def pyproject_to_conda( sort=sort, deps=deps, reqs=reqs, + remove_whitespace=remove_whitespace, ) return _output_to_yaml( **output, @@ -552,6 +567,7 @@ def to_conda_yaml( deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, allow_empty: bool = False, + remove_whitespace: bool = True, ) -> str: self._check_extras(extras) @@ -569,6 +585,7 @@ def to_conda_yaml( deps=deps, reqs=reqs, allow_empty=allow_empty, + remove_whitespace=remove_whitespace, ) def to_conda_lists( @@ -581,6 +598,7 @@ def to_conda_lists( sort: bool = True, deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, + remove_whitespace: bool = True, ) -> dict[str, Any]: self._check_extras(extras) @@ -594,6 +612,7 @@ def to_conda_lists( sort=sort, deps=deps, reqs=reqs, + remove_whitespace=remove_whitespace, ) def to_requirement_list( @@ -602,6 +621,7 @@ def to_requirement_list( include_base_dependencies: bool = True, sort: bool = True, reqs: Sequence[str] | None = None, + remove_whitespace: bool = True, ) -> list[str]: self._check_extras(extras) @@ -610,11 +630,14 @@ def to_requirement_list( extras=extras, include_base_dependencies=include_base_dependencies, ) - out = [x for x, y in values if x is not None] + out = [x for x, _ in values if x is not None] if reqs: out.extend(list(reqs)) + if remove_whitespace: + out = [_remove_whitespace(x) for x in out] + if sort: return sorted(out) else: @@ -629,6 +652,7 @@ def to_requirements( sort: bool = True, reqs: Sequence[str] | None = None, allow_empty: bool = False, + remove_whitespace: bool = True, ) -> str: """Create requirements.txt like file with pip dependencies.""" @@ -639,6 +663,7 @@ def to_requirements( include_base_dependencies=include_base_dependencies, sort=sort, reqs=reqs, + remove_whitespace=remove_whitespace, ) if not reqs: @@ -662,6 +687,7 @@ def to_conda_requirements( sort: bool = True, deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, + remove_whitespace: bool = True, ) -> tuple[str, str]: output = self.to_conda_lists( extras=extras, @@ -672,6 +698,7 @@ def to_conda_requirements( sort=sort, deps=deps, reqs=reqs, + remove_whitespace=remove_whitespace, ) deps = output.get("dependencies", None) diff --git a/src/pyproject2conda/utils.py b/src/pyproject2conda/utils.py index 78290b0..8561ebe 100644 --- a/src/pyproject2conda/utils.py +++ b/src/pyproject2conda/utils.py @@ -88,7 +88,7 @@ def update_target( update = True else: # check times - deps_filtered = [] + deps_filtered: list[Path] = [] for d in map(Path, deps): if d.exists(): deps_filtered.append(d) @@ -96,6 +96,8 @@ def update_target( target_time = target.stat().st_mtime update = any(target_time < dep.stat().st_mtime for dep in deps_filtered) + else: + raise ValueError(f"unknown option overwrite={overwrite}") return update @@ -121,7 +123,7 @@ def filename_from_template( if template is None: return None - kws = {} + kws: dict[str, str] = {} if python: py_version = python elif python_version: diff --git a/tests/data/test-pyproject.toml b/tests/data/test-pyproject.toml index ff49dee..808b0f8 100644 --- a/tests/data/test-pyproject.toml +++ b/tests/data/test-pyproject.toml @@ -40,6 +40,18 @@ python = ["3.10"] user_config = "config/userconfig.toml" default_envs = ["test", "dev", "dist-pypi"] +[tool.pyproject2conda.envs.base] +style = ["requirements"] +# Note that the default value for `extras` is the name of the environment. +# To have no extras, either pass +# extras = [] +# or +# +extras = false + +# +# A value of `extras = true` also implies using the environment name +# as the extras. [tool.pyproject2conda.envs."test-extras"] extras = ["test"] style = ["yaml", "requirements"] diff --git a/tests/test_cli.py b/tests/test_cli.py index 426d8b4..c92166e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -353,7 +353,7 @@ def test_requirements(): expected = """\ athing bthing -cthing; python_version < '3.10' +cthing;python_version<'3.10' """ for cmd in ["r", "requirements"]: @@ -363,7 +363,7 @@ def test_requirements(): expected = """\ athing bthing -cthing; python_version < '3.10' +cthing;python_version<'3.10' pandas pytest matplotlib @@ -378,7 +378,7 @@ def test_requirements(): expected = """\ athing bthing -cthing; python_version < '3.10' +cthing;python_version<'3.10' pandas pytest matplotlib @@ -403,7 +403,7 @@ def test_requirements(): expected = """\ athing bthing -cthing; python_version < '3.10' +cthing;python_version<'3.10' matplotlib pandas pytest @@ -418,7 +418,7 @@ def test_requirements(): expected = """\ athing bthing -cthing; python_version < '3.10' +cthing;python_version<'3.10' matplotlib other pandas @@ -439,6 +439,32 @@ def test_requirements(): check_result(result, expected) + # allow whitespace: + expected = """\ +athing +bthing +cthing; python_version < '3.10' +matplotlib +other +pandas +pytest +thing; python_version < '3.10' + """ + + result = do_run( + runner, + "requirements", + "-e", + "dev", + "-r", + "thing; python_version < '3.10'", + "-r", + "other", + "--no-remove-whitespace", + ) + + check_result(result, expected) + expected = """\ setuptools build diff --git a/tests/test_config.py b/tests/test_config.py index b26d5d9..a5909c4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -78,6 +78,10 @@ def test_option_override(): extras = [] style = "yaml" python = [] + + [tool.pyproject2conda.envs.base2] + style = "yaml" + extras = [] """ d = Config.from_string(dedent(toml)) @@ -98,10 +102,33 @@ def test_option_override(): "name": None, "channels": ["conda-forge"], "allow_empty": False, + "remove_whitespace": True, "output": "hello-base.yaml", }, ) + output = list(d.iter(envs=["base2"])) + + assert output[0] == ( + "yaml", + { + "extras": [], + "sort": True, + "base": True, + "header": None, + "overwrite": "check", + "verbose": None, + "reqs": None, + "deps": None, + "name": None, + "channels": ["conda-forge"], + "allow_empty": False, + "remove_whitespace": True, + "output": "py310-base2.yaml", + "python": "3.10", + }, + ) + output = list(d.iter(envs=["base"], template="there-{env}")) assert output[0] == ( @@ -118,6 +145,7 @@ def test_option_override(): "name": None, "channels": ["conda-forge"], "allow_empty": False, + "remove_whitespace": True, "output": "there-base.yaml", }, ) @@ -138,6 +166,35 @@ def test_option_override(): "name": None, "channels": ["conda-forge"], "allow_empty": True, + "remove_whitespace": True, + "output": "there-base.yaml", + }, + ) + + output = list( + d.iter( + envs=["base"], + allow_empty=True, + remove_whitespace=False, + template="there-{env}", + ) + ) + + assert output[0] == ( + "yaml", + { + "extras": [], + "sort": True, + "base": True, + "header": None, + "overwrite": "check", + "verbose": None, + "reqs": None, + "deps": None, + "name": None, + "channels": ["conda-forge"], + "allow_empty": True, + "remove_whitespace": False, "output": "there-base.yaml", }, ) @@ -192,6 +249,7 @@ def test_config_only_default(): "deps": None, "reqs": None, "allow_empty": False, + "remove_whitespace": True, }, ) ] @@ -206,6 +264,7 @@ def test_config_only_default(): python = ["3.8"] [tool.pyproject2conda.envs.test] + extras = true """ s2 = """ @@ -251,6 +310,7 @@ def test_config_overrides(): "deps": None, "reqs": None, "allow_empty": False, + "remove_whitespace": True, }, ) @@ -308,6 +368,7 @@ def test_config_python_include_version(): "deps": None, "reqs": None, "allow_empty": False, + "remove_whitespace": True, }, ), ( @@ -327,6 +388,7 @@ def test_config_python_include_version(): "deps": None, "reqs": None, "allow_empty": False, + "remove_whitespace": True, }, ), ] @@ -374,6 +436,7 @@ def test_config_user_config(): "deps": None, "reqs": None, "allow_empty": False, + "remove_whitespace": True, }, ), ( @@ -392,6 +455,7 @@ def test_config_user_config(): "deps": None, "reqs": None, "allow_empty": False, + "remove_whitespace": True, }, ), ] @@ -516,11 +580,13 @@ def test_multiple(): f"{path2}/py310-user-dev.yaml", ) + do_run(runner, "req", "-o", f"{path2}/base.txt") + paths1 = Path(path1).glob("*") names1 = set(x.name for x in paths1) expected = set( - "py310-dev.yaml py310-dist-pypi.yaml py310-test-extras.yaml py310-test.yaml py310-user-dev.yaml py311-test-extras.yaml py311-test.yaml test-extras.txt".split() + "base.txt py310-dev.yaml py310-dist-pypi.yaml py310-test-extras.yaml py310-test.yaml py310-user-dev.yaml py311-test-extras.yaml py311-test.yaml test-extras.txt".split() ) assert names1 == expected diff --git a/tests/test_parser.py b/tests/test_parser.py index 05e4cf3..399c549 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -189,11 +189,50 @@ def test_output_to_yaml(): assert s == dedent(expected) -def test_complete(): +def test_infer(): toml = dedent( """\ [project] name = "hello" + dependencies = [ + "athing", # p2c: -p # a comment + "bthing", # p2c: -s bthing-conda + "cthing; python_version<'3.10'", # p2c: -c conda-forge + ] + + [project.optional-dependencies] + test = [ + "pandas", + "pytest", # p2c: -c conda-forge + ] + dev-extras = [ + # p2c: -s additional-thing # this is an additional conda package + "matplotlib", # p2c: -s conda-matplotlib + ] + dev = [ + "hello[test]", + "hello[dev-extras]", + ] + dist-pypi = [ + "setuptools", + "build", # p2c: -p + ] + + + [tool.pyproject2conda] + channels = ['conda-forge'] + """ + ) + + d = parser.PyProject2Conda.from_string(toml) + with pytest.raises(ValueError): + d.to_conda_yaml(python_include="infer") + + +def test_package_name(): + toml = dedent( + """\ + [project] requires-python = ">=3.8,<3.11" dependencies = [ "athing", # p2c: -p # a comment @@ -220,6 +259,46 @@ def test_complete(): ] + [tool.pyproject2conda] + channels = ['conda-forge'] + """ + ) + d = parser.PyProject2Conda.from_string(toml) + with pytest.raises(ValueError): + d.to_conda_lists() + + +def test_complete(): + toml = dedent( + """\ + [project] + name = "hello" + requires-python = ">=3.8, <3.11" + dependencies = [ + "athing", # p2c: -p # a comment + "bthing", # p2c: -s bthing-conda + "cthing; python_version<'3.10'", # p2c: -c conda-forge + ] + + [project.optional-dependencies] + test = [ + "pandas", + "pytest", # p2c: -c conda-forge + ] + dev-extras = [ + # p2c: -s additional-thing # this is an additional conda package + "matplotlib", # p2c: -s conda-matplotlib + ] + dev = [ + "hello[test]", + "hello[dev-extras]", + ] + dist-pypi = [ + "setuptools", + "build", # p2c: -p + ] + + [tool.pyproject2conda] channels = ['conda-forge'] """ @@ -263,6 +342,22 @@ def test_complete(): """ assert dedent(expected) == d.to_conda_yaml(python_include="infer") + # with remove spaces: + expected = """\ +channels: + - conda-forge +dependencies: + - python>=3.8, <3.11 + - bthing-conda + - conda-forge::cthing + - pip + - pip: + - athing + """ + assert dedent(expected) == d.to_conda_yaml( + python_include="infer", remove_whitespace=False + ) + expected = """\ channels: - conda-forge @@ -516,3 +611,70 @@ def test_missing_dependencies(): f() assert f(allow_empty=True) == "No dependencies for this environment\n" + + +def test_spaces(): + toml = dedent( + """\ + [project] + name = "hello" + requires-python = ">=3.8, <3.11" + dependencies = [ + "athing >0.5", # p2c: -p # a comment + "bthing > 1.0", # p2c: -s 'bthing-conda > 2.0' + "cthing; python_version < '3.10'", # p2c: -c conda-forge + ] + + + [tool.pyproject2conda] + channels = ['conda-forge'] + """ + ) + + d = parser.PyProject2Conda.from_string(toml) + + expected = """\ +channels: + - conda-forge +dependencies: + - python>=3.8, <3.11 + - bthing-conda > 2.0 + - conda-forge::cthing + - pip + - pip: + - athing >0.5 + """ + assert dedent(expected) == d.to_conda_yaml( + python_include="infer", remove_whitespace=False + ) + + expected = """\ +channels: + - conda-forge +dependencies: + - python>=3.8,<3.11 + - bthing-conda>2.0 + - conda-forge::cthing + - pip + - pip: + - athing>0.5 + """ + assert dedent(expected) == d.to_conda_yaml( + python_include="infer", remove_whitespace=True + ) + + expected = """\ + athing >0.5 + bthing > 1.0 + cthing; python_version < '3.10' + """ + + assert dedent(expected) == d.to_requirements(remove_whitespace=False) + + expected = """\ + athing>0.5 + bthing>1.0 + cthing;python_version<'3.10' + """ + + assert dedent(expected) == d.to_requirements(remove_whitespace=True) diff --git a/tools/create_pythons.py b/tools/create_pythons.py index 1be3ec5..c075ea0 100644 --- a/tools/create_pythons.py +++ b/tools/create_pythons.py @@ -1,8 +1,11 @@ """Script to create pythons for use with virtualenvs""" from __future__ import annotations +import sys from functools import lru_cache +assert sys.version_info >= (3, 9) + @lru_cache def conda_cmd() -> str: diff --git a/tools/noxtools.py b/tools/noxtools.py index 1ad3d8a..afe84e7 100644 --- a/tools/noxtools.py +++ b/tools/noxtools.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Iterable, Literal, Sequence, TextIO, cast -from ruamel.yaml import safe_load +from ruamel.yaml import YAML if TYPE_CHECKING: from collections.abc import Collection @@ -60,6 +60,7 @@ def pkg_install_condaenv( display_name: str | None = None, install_package: bool = True, update: bool = False, + update_package: bool = False, log_session: bool = False, deps: Collection[str] | None = None, reqs: Collection[str] | None = None, @@ -86,6 +87,7 @@ def check_filename(filename: str | Path) -> str: lockfile=check_filename(filename), display_name=display_name, update=update, + update_package=update_package, install_package=install_package, **kwargs, ) @@ -96,6 +98,7 @@ def check_filename(filename: str | Path) -> str: check_filename(filename), display_name=display_name, update=update, + update_package=update_package, deps=deps, reqs=reqs, channels=channels, @@ -117,6 +120,7 @@ def pkg_install_venv( reqs: Collection[str] | None = None, display_name: str | None = None, update: bool = False, + update_package: bool = False, install_package: bool = False, no_deps: bool = True, log_session: bool = False, @@ -132,6 +136,7 @@ def pkg_install_venv( reqs=reqs, display_name=display_name, update=update, + update_package=update_package, install_package=install_package, no_deps=no_deps, lock=lock, @@ -317,6 +322,7 @@ def session_install_envs_lock( extras: str | list[str] | None = None, display_name: str | None = None, update: bool = False, + update_package: bool = False, install_package: bool = False, ) -> bool: """Install dependencies using conda-lock.""" @@ -327,34 +333,36 @@ def session_install_envs_lock( unchanged, hashes = env_unchanged( session, lockfile, prefix="lock", other=dict(install_package=install_package) ) - if unchanged and not update: - return unchanged - if extras: - if isinstance(extras, str): - extras = extras.split(",") - extras = cast(list[str], sum([["--extras", _] for _ in extras], [])) - else: - extras = [] - - session.run( - "conda-lock", - "install", - "--mamba", - *extras, - "-p", - str(session.virtualenv.location), - str(lockfile), - silent=True, - external=True, - ) + do_dep = update or (not unchanged) + do_pkg = install_package and (do_dep or update_package) - if install_package: - session_install_package(session) + if do_dep: + if extras: + if isinstance(extras, str): + extras = extras.split(",") + extras = cast(list[str], sum([["--extras", _] for _ in extras], [])) + else: + extras = [] + + session.run( + "conda-lock", + "install", + "--mamba", + *extras, + "-p", + str(session.virtualenv.location), + str(lockfile), + silent=True, + external=True, + ) - session_set_ipykernel_display_name(session, display_name) + if do_pkg: + session_install_package(session) - write_hashfile(hashes, session=session, prefix="lock") + if do_dep or do_pkg: + session_set_ipykernel_display_name(session, display_name) + write_hashfile(hashes, session=session, prefix="lock") return unchanged @@ -394,7 +402,7 @@ def _get_context(path: str | Path | TextIO) -> TextIO | Path: for path in paths: with _get_context(path) as f: - data = safe_load(f) + data = YAML(typ="safe", pure=True).load(f) channels.update(data.get("channels", [])) name = data.get("name", name) @@ -421,6 +429,7 @@ def session_install_envs( install_kws: dict[str, Any] | None = None, display_name: str | None = None, update: bool = False, + update_package: bool = False, install_package: bool = False, ) -> bool: """Parse and install everything. Pass an already merged yaml file.""" @@ -446,30 +455,32 @@ def session_install_envs( install_package=install_package, ), ) - if unchanged and not update: - return unchanged - if not channels: - channels = "" - if deps: - conda_install_kws = conda_install_kws or {} - conda_install_kws.update(channel=channels) - if update: - deps = ["--update-all"] + list(deps) + do_dep = update or (not unchanged) + do_pkg = install_package and (do_dep or update_package) - session.conda_install(*deps, **(conda_install_kws or {})) + if do_dep: + if not channels: + channels = "" + if deps: + conda_install_kws = conda_install_kws or {} + conda_install_kws.update(channel=channels) + if update: + deps = ["--update-all"] + list(deps) - if reqs: - if update: - reqs = ["--upgrade"] + list(reqs) - session.install(*reqs, **(install_kws or {})) + session.conda_install(*deps, **(conda_install_kws or {})) - if install_package: - session_install_package(session) + if reqs: + if update: + reqs = ["--upgrade"] + list(reqs) + session.install(*reqs, **(install_kws or {})) - session_set_ipykernel_display_name(session, display_name) + if do_pkg: + session_install_package(session) - write_hashfile(hashes, session=session, prefix="env") + if do_dep or do_pkg: + session_set_ipykernel_display_name(session, display_name) + write_hashfile(hashes, session=session, prefix="env") return unchanged @@ -483,6 +494,7 @@ def session_install_pip( reqs: str | Collection[str] | None = None, display_name: str | None = None, update: bool = False, + update_package: bool = False, install_package: bool = False, no_deps: bool = True, lock: bool = False, @@ -545,26 +557,28 @@ def _verify_paths(paths: str | list[str]) -> list[str]: ), ) - if unchanged and not update: - return unchanged + do_dep = update or (not unchanged) + do_pkg = install_package and (do_dep or update_package) - # do install - install_args = ( - prepend_flag("-r", *requirement_paths) - + prepend_flag("-c", *constraint_paths) - + list(reqs) - ) + if do_dep: + # do install + install_args = ( + prepend_flag("-r", *requirement_paths) + + prepend_flag("-c", *constraint_paths) + + list(reqs) + ) - if install_args: - if update: - install_args = ["--upgrade"] + list(install_args) - session.install(*install_args) + if install_args: + if update: + install_args = ["--upgrade"] + list(install_args) + session.install(*install_args) - if install_package: + if do_pkg: session.install(*install_package_args) - session_set_ipykernel_display_name(session, display_name) - write_hashfile(hashes, session=session, prefix="pip") + if do_dep or do_pkg: + session_set_ipykernel_display_name(session, display_name) + write_hashfile(hashes, session=session, prefix="pip") return unchanged