From 792e559b23422533867e1ee21ec4248048468a01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Thu, 6 Apr 2023 15:53:18 -0700 Subject: [PATCH] Order depends (#143) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- src/tox_ini_fmt/formatter/section_order.py | 2 +- src/tox_ini_fmt/formatter/test_env.py | 58 +--------- src/tox_ini_fmt/formatter/tox_section.py | 72 +----------- src/tox_ini_fmt/formatter/util.py | 122 +++++++++++++++++++++ tests/formatter/test_test_env.py | 9 +- tests/formatter/test_tox_section.py | 2 +- 6 files changed, 136 insertions(+), 129 deletions(-) diff --git a/src/tox_ini_fmt/formatter/section_order.py b/src/tox_ini_fmt/formatter/section_order.py index 2bb944b..5ddb7e0 100644 --- a/src/tox_ini_fmt/formatter/section_order.py +++ b/src/tox_ini_fmt/formatter/section_order.py @@ -3,7 +3,7 @@ import itertools from configparser import ConfigParser -from .tox_section import order_env_list +from .util import order_env_list def order_sections(parser: ConfigParser, pin_toxenvs: list[str]) -> None: diff --git a/src/tox_ini_fmt/formatter/test_env.py b/src/tox_ini_fmt/formatter/test_env.py index 649521b..7aa23a1 100644 --- a/src/tox_ini_fmt/formatter/test_env.py +++ b/src/tox_ini_fmt/formatter/test_env.py @@ -1,13 +1,10 @@ from __future__ import annotations -import itertools -import re -from collections import defaultdict from configparser import ConfigParser +from functools import partial from typing import Callable, Mapping -from .requires import requires -from .util import fix_and_reorder, is_substitute, to_boolean +from .util import collect_multi_line, fix_and_reorder, fmt_list, to_boolean, to_list_of_env_values, to_py_dependencies def format_test_env(parser: ConfigParser, name: str) -> None: @@ -52,50 +49,11 @@ def format_test_env(parser: ConfigParser, name: str) -> None: "suicide_timeout": str, "interrupt_timeout": str, "terminate_timeout": str, - "depends": to_ordered_list, + "depends": partial(to_list_of_env_values, []), } fix_and_reorder(parser, name, tox_section_cfg) -CONDITIONAL_MARKER = re.compile(r"(?P[a-zA-Z0-9, ]+):(?P.*)") - - -def collect_multi_line( - value: str, - line_split: str | None = r",| |\t", - normalize: Callable[[dict[str, list[str]]], dict[str, list[str]]] | None = None, - sort_key: Callable[[str], str] | None = None, -) -> tuple[list[str], list[str]]: - groups: defaultdict[str, list[str]] = defaultdict(list) - substitute: list[str] = [] - for line in value.strip().splitlines(): - match = CONDITIONAL_MARKER.match(line) - if match: - elements = match.groupdict() - normalized_key = ", ".join(sorted(i.strip() for i in elements["envs"].split(","))) - groups[normalized_key].append(elements["value"].strip()) - else: - for part in re.split(line_split, line.strip()) if line_split else [line.strip()]: - if part: # remove empty lines - if is_substitute(part): - substitute.append(part) - else: - if part not in groups[""]: # remove duplicates - groups[""].append(part) - normalized_group = normalize(groups) if normalize else groups - result = list( - itertools.chain.from_iterable( - (f"{k}: {d}" if k != "" else d for d in sorted(v, key=sort_key)) - for k, v in sorted(normalized_group.items(), key=lambda i: (len(i[0].split(", ")), i[0])) - ), - ) - return result, substitute - - -def fmt_list(values: list[str], substitute: list[str]) -> str: - return "\n".join([""] + sorted(substitute) + values) - - def to_ordered_list(value: str) -> str: """Must be a line separated list - fix comma separated format""" extras, substitute = collect_multi_line(value) @@ -135,13 +93,3 @@ def to_commands(value: str) -> str: result.append(f"{prepend}{val}{ending}") ends_with_sep = cur_ends_with_sep return fmt_list(result, []) - - -def to_py_dependencies(value: str) -> str: - raw_deps, substitute = collect_multi_line( - value, - line_split=None, - normalize=lambda groups: {k: requires(v) for k, v in groups.items()}, - sort_key=lambda _: "", # noqa: U101 # we already sorted as we wanted in normalize, keep it as is - ) - return fmt_list(raw_deps, substitute) diff --git a/src/tox_ini_fmt/formatter/tox_section.py b/src/tox_ini_fmt/formatter/tox_section.py index 16b7b6c..8b27e38 100644 --- a/src/tox_ini_fmt/formatter/tox_section.py +++ b/src/tox_ini_fmt/formatter/tox_section.py @@ -1,12 +1,10 @@ from __future__ import annotations -import re from configparser import ConfigParser from functools import partial from typing import Callable, Mapping -from .test_env import to_py_dependencies -from .util import fix_and_reorder, to_boolean +from .util import fix_and_reorder, to_boolean, to_list_of_env_values, to_py_dependencies def format_tox_section(parser: ConfigParser, pin_toxenvs: list[str]) -> None: @@ -29,71 +27,3 @@ def format_tox_section(parser: ConfigParser, pin_toxenvs: list[str]) -> None: "ignore_basepython_conflict": to_boolean, } fix_and_reorder(parser, "tox", tox_section_cfg) - - -def to_list_of_env_values(pin_toxenvs: list[str], payload: str) -> str: - """ - Example: - - envlist = py39,py38 - envlist = {py37,py36}-django{20,21},{py37,py36}-mango{20,21},py38 - """ - within_braces, values = False, [] - cur_str, brace_str = "", "" - for char in payload: - if char == "{": - within_braces = True - elif char == "}": - within_braces = False - envs = [i.strip() for i in brace_str[1:].split(",")] - order_env_list(envs, pin_toxenvs) - cur_str += f'{{{", ".join(envs)}}}' - brace_str = "" - continue - elif char in (",", "\n"): - if within_braces: - pass - else: - to_add = cur_str.strip() - if to_add: - values.append(to_add) - cur_str = "" - continue - if within_braces: - brace_str += char - else: - cur_str += char - # avoid adding an empty value, caused e.g. by a trailing comma - last_entry = cur_str.strip() - if last_entry != "": - values.append(last_entry) - # start with higher python version - order_env_list(values, pin_toxenvs) - # use newline instead of comma as separator, indent values one per newline (no value on key-row) - result = "\n{}".format("\n".join(f"{v}" for v in values)) - return result - - -def order_env_list(values: list[str], pin_toxenvs: list[str]) -> None: - values.sort(key=partial(_get_py_version, pin_toxenvs), reverse=True) - - -_MATCHER = re.compile(r"^([a-zA-Z]*)(\d*)$") - - -def _get_py_version(pin_toxenvs: list[str], env_list: str) -> tuple[int, int]: - for element in env_list.split("-"): - if element in pin_toxenvs: - return len(element) - pin_toxenvs.index(element), 0 - match = _MATCHER.match(element) - if match is not None: - name, version = match.groups() - name = name.lower() - if name == "py": - main = 0 - elif name == "pypy": - main = -1 - else: - main = -2 - return main, int(version) if version else 0 - return -3, 0 diff --git a/src/tox_ini_fmt/formatter/util.py b/src/tox_ini_fmt/formatter/util.py index 7979e7c..ffa8f68 100644 --- a/src/tox_ini_fmt/formatter/util.py +++ b/src/tox_ini_fmt/formatter/util.py @@ -1,9 +1,14 @@ from __future__ import annotations +import itertools import re +from collections import defaultdict from configparser import ConfigParser +from functools import partial from typing import Callable, Mapping +from .requires import requires + def to_boolean(payload: str) -> str: return "true" if payload.lower() == "true" else "false" @@ -38,3 +43,120 @@ def is_substitute(value: str) -> bool: sub_key = match.group("substitution_value") return sub_key.startswith("[") and "]" in sub_key return False + + +_MATCHER = re.compile(r"^([a-zA-Z]*)(\d*)$") + + +def to_list_of_env_values(pin_toxenvs: list[str], payload: str) -> str: + """ + Example: + + envlist = py39,py38 + envlist = {py37,py36}-django{20,21},{py37,py36}-mango{20,21},py38 + """ + within_braces, values = False, [] + cur_str, brace_str = "", "" + for char in payload: + if char == "{": + within_braces = True + elif char == "}": + within_braces = False + envs = [i.strip() for i in brace_str[1:].split(",")] + order_env_list(envs, pin_toxenvs) + cur_str += f'{{{", ".join(envs)}}}' + brace_str = "" + continue + elif char in (",", "\n"): + if within_braces: + pass + else: + to_add = cur_str.strip() + if to_add: + values.append(to_add) + cur_str = "" + continue + if within_braces: + brace_str += char + else: + cur_str += char + # avoid adding an empty value, caused e.g. by a trailing comma + last_entry = cur_str.strip() + if last_entry != "": + values.append(last_entry) + # start with higher python version + order_env_list(values, pin_toxenvs) + # use newline instead of comma as separator, indent values one per newline (no value on key-row) + result = "\n{}".format("\n".join(f"{v}" for v in values)) + return result + + +def _get_py_version(pin_toxenvs: list[str], env_list: str) -> tuple[int, int]: + for element in env_list.split("-"): + if element in pin_toxenvs: + return len(element) - pin_toxenvs.index(element), 0 + match = _MATCHER.match(element) + if match is not None: + name, version = match.groups() + name = name.lower() + if name == "py": + main = 0 + elif name == "pypy": + main = -1 + else: + main = -2 + return main, int(version) if version else 0 + return -3, 0 + + +def order_env_list(values: list[str], pin_toxenvs: list[str]) -> None: + values.sort(key=partial(_get_py_version, pin_toxenvs), reverse=True) + + +CONDITIONAL_MARKER = re.compile(r"(?P[a-zA-Z0-9, ]+):(?P.*)") + + +def collect_multi_line( + value: str, + line_split: str | None = r",| |\t", + normalize: Callable[[dict[str, list[str]]], dict[str, list[str]]] | None = None, + sort_key: Callable[[str], str] | None = None, +) -> tuple[list[str], list[str]]: + groups: defaultdict[str, list[str]] = defaultdict(list) + substitute: list[str] = [] + for line in value.strip().splitlines(): + match = CONDITIONAL_MARKER.match(line) + if match: + elements = match.groupdict() + normalized_key = ", ".join(sorted(i.strip() for i in elements["envs"].split(","))) + groups[normalized_key].append(elements["value"].strip()) + else: + for part in re.split(line_split, line.strip()) if line_split else [line.strip()]: + if part: # remove empty lines + if is_substitute(part): + substitute.append(part) + else: + if part not in groups[""]: # remove duplicates + groups[""].append(part) + normalized_group = normalize(groups) if normalize else groups + result = list( + itertools.chain.from_iterable( + (f"{k}: {d}" if k != "" else d for d in sorted(v, key=sort_key)) + for k, v in sorted(normalized_group.items(), key=lambda i: (len(i[0].split(", ")), i[0])) + ), + ) + return result, substitute + + +def to_py_dependencies(value: str) -> str: + raw_deps, substitute = collect_multi_line( + value, + line_split=None, + normalize=lambda groups: {k: requires(v) for k, v in groups.items()}, + sort_key=lambda _: "", # noqa: U101 # we already sorted as we wanted in normalize, keep it as is + ) + return fmt_list(raw_deps, substitute) + + +def fmt_list(values: list[str], substitute: list[str]) -> str: + return "\n".join([""] + sorted(substitute) + values) diff --git a/tests/formatter/test_test_env.py b/tests/formatter/test_test_env.py index d15c495..1a26e78 100644 --- a/tests/formatter/test_test_env.py +++ b/tests/formatter/test_test_env.py @@ -6,7 +6,8 @@ import pytest from tox_ini_fmt.formatter import format_tox_ini -from tox_ini_fmt.formatter.test_env import to_ordered_list, to_py_dependencies +from tox_ini_fmt.formatter.test_env import to_ordered_list +from tox_ini_fmt.formatter.util import to_py_dependencies def test_no_tox_section(tox_ini: Path) -> None: @@ -155,3 +156,9 @@ def test_deps_conditional() -> None: def test_python_req_sort_by_name() -> None: result = to_py_dependencies("pytest-cov\npytest\npytest-magic>=1\npytest>=1") assert result == "\npytest\npytest>=1\npytest-cov\npytest-magic>=1" + + +def test_depends_ordering(tox_ini: Path) -> None: + tox_ini.write_text("[testenv]\ndepends =\n py311\n py312\n py39\n p310") + outcome = format_tox_ini(tox_ini) + assert outcome == "[testenv]\ndepends =\n py312\n py311\n py39\n p310\n" diff --git a/tests/formatter/test_tox_section.py b/tests/formatter/test_tox_section.py index f4d60d3..23ecf27 100644 --- a/tests/formatter/test_tox_section.py +++ b/tests/formatter/test_tox_section.py @@ -6,7 +6,7 @@ import pytest from tox_ini_fmt.formatter import format_tox_ini -from tox_ini_fmt.formatter.tox_section import order_env_list +from tox_ini_fmt.formatter.util import order_env_list def test_no_tox_section(tox_ini: Path) -> None: