diff --git a/mypy/config_parser.py b/mypy/config_parser.py index 0e033471d2e9..d6836b096aac 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -16,7 +16,7 @@ import tomli as tomllib from collections.abc import Mapping, MutableMapping, Sequence -from typing import Any, Callable, Final, TextIO, Union +from typing import Any, Callable, Final, TextIO, Union, cast from typing_extensions import TypeAlias as _TypeAlias from mypy import defaults @@ -292,7 +292,7 @@ def parse_config_file( stdout: TextIO | None = None, stderr: TextIO | None = None, ) -> None: - """Parse a config file into an Options object. + """Parse a config file into an Options object, following config extend arguments. Errors are written to stderr but are not fatal. @@ -301,36 +301,128 @@ def parse_config_file( stdout = stdout or sys.stdout stderr = stderr or sys.stderr + strict_found = False + + def set_strict(value: bool) -> None: + nonlocal strict_found + strict_found = value + + ret = _parse_and_extend_config_file( + template=options, + set_strict=set_strict, + filename=filename, + stdout=stdout, + stderr=stderr, + visited=set(), + ) + + if ret is None: + return + + file_read, mypy_updates, mypy_report_dirs, module_updates = ret + + if strict_found: + set_strict_flags() + + options.config_file = file_read + + for k, v in mypy_updates.items(): + setattr(options, k, v) + + options.report_dirs.update(mypy_report_dirs) + + for glob, updates in module_updates.items(): + options.per_module_options[glob] = updates + + +def _merge_updates(existing: dict[str, object], new: dict[str, object]) -> None: + existing["disable_error_code"] = list( + set( + cast(list[str], existing.get("disable_error_code", [])) + + cast(list[str], new.pop("disable_error_code", [])) + ) + ) + existing["enable_error_code"] = list( + set( + cast(list[str], existing.get("enable_error_code", [])) + + cast(list[str], new.pop("enable_error_code", [])) + ) + ) + existing.update(new) + + +def _parse_and_extend_config_file( + template: Options, + set_strict: Callable[[bool], None], + filename: str | None, + stdout: TextIO, + stderr: TextIO, + visited: set[str], +) -> tuple[str, dict[str, object], dict[str, str], dict[str, dict[str, object]]] | None: ret = ( _parse_individual_file(filename, stderr) if filename is not None else _find_config_file(stderr) ) if ret is None: - return + return None parser, config_types, file_read = ret - options.config_file = file_read - os.environ["MYPY_CONFIG_FILE_DIR"] = os.path.dirname(os.path.abspath(file_read)) + abs_file_read = os.path.abspath(file_read) + if abs_file_read in visited: + print(f"Circular extend detected: {abs_file_read}", file=stderr) + return None + visited.add(abs_file_read) + + if len(visited) == 1: + # set it only after the first config file is visited to allow for path variable expansions + # when parsing below, so recursive calls for config extend references won't overwrite it + os.environ["MYPY_CONFIG_FILE_DIR"] = os.path.dirname(abs_file_read) + + mypy_updates: dict[str, object] = {} + mypy_report_dirs: dict[str, str] = {} + module_updates: dict[str, dict[str, object]] = {} if "mypy" not in parser: if filename or os.path.basename(file_read) not in defaults.SHARED_CONFIG_NAMES: print(f"{file_read}: No [mypy] section in config file", file=stderr) else: section = parser["mypy"] + + extend = section.pop("extend", None) + if extend: + parse_ret = _parse_and_extend_config_file( + template=template, + set_strict=set_strict, + # refer to extend relative to directory where we found current config + filename=os.path.relpath( + os.path.normpath( + os.path.join(os.path.dirname(abs_file_read), expand_path(extend)) + ) + ), + stdout=stdout, + stderr=stderr, + visited=visited, + ) + + if parse_ret is None: + print(f"{extend} is not a valid path to extend from {abs_file_read}", file=stderr) + else: + _, mypy_updates, mypy_report_dirs, module_updates = parse_ret + prefix = f"{file_read}: [mypy]: " updates, report_dirs = parse_section( - prefix, options, set_strict_flags, section, config_types, stderr + prefix, template, set_strict, section, config_types, stderr ) - for k, v in updates.items(): - setattr(options, k, v) - options.report_dirs.update(report_dirs) + # extend and overwrite existing values with new ones + _merge_updates(mypy_updates, updates) + mypy_report_dirs.update(report_dirs) for name, section in parser.items(): if name.startswith("mypy-"): prefix = get_prefix(file_read, name) updates, report_dirs = parse_section( - prefix, options, set_strict_flags, section, config_types, stderr + prefix, template, set_strict, section, config_types, stderr ) if report_dirs: print( @@ -367,7 +459,10 @@ def parse_config_file( file=stderr, ) else: - options.per_module_options[glob] = updates + # extend and overwrite existing values with new ones + _merge_updates(module_updates.setdefault(glob, {}), updates) + + return file_read, mypy_updates, mypy_report_dirs, module_updates def get_prefix(file_read: str, name: str) -> str: @@ -469,7 +564,7 @@ def destructure_overrides(toml_data: dict[str, Any]) -> dict[str, Any]: def parse_section( prefix: str, template: Options, - set_strict_flags: Callable[[], None], + set_strict: Callable[[bool], None], section: Mapping[str, Any], config_types: dict[str, Any], stderr: TextIO = sys.stderr, @@ -558,8 +653,7 @@ def parse_section( print(f"{prefix}{key}: {err}", file=stderr) continue if key == "strict": - if v: - set_strict_flags() + set_strict(v) continue results[options_key] = v @@ -660,12 +754,12 @@ def parse_mypy_comments( stderr = StringIO() strict_found = False - def set_strict_flags() -> None: + def set_strict(value: bool) -> None: nonlocal strict_found - strict_found = True + strict_found = value new_sections, reports = parse_section( - "", template, set_strict_flags, parser["dummy"], ini_config_types, stderr=stderr + "", template, set_strict, parser["dummy"], ini_config_types, stderr=stderr ) errors.extend((lineno, x) for x in stderr.getvalue().strip().split("\n") if x) if reports: diff --git a/mypy/test/test_config_parser.py b/mypy/test/test_config_parser.py index 597143738f23..e6f3af97bd42 100644 --- a/mypy/test/test_config_parser.py +++ b/mypy/test/test_config_parser.py @@ -1,14 +1,16 @@ from __future__ import annotations import contextlib +import io import os import tempfile import unittest from collections.abc import Iterator from pathlib import Path -from mypy.config_parser import _find_config_file +from mypy.config_parser import _find_config_file, parse_config_file from mypy.defaults import CONFIG_NAMES, SHARED_CONFIG_NAMES +from mypy.options import Options @contextlib.contextmanager @@ -128,3 +130,124 @@ def test_precedence_missing_section(self) -> None: result = _find_config_file() assert result is not None assert Path(result[2]).resolve() == parent_mypy.resolve() + + +class ExtendConfigFileSuite(unittest.TestCase): + + def test_extend_success(self) -> None: + with tempfile.TemporaryDirectory() as _tmpdir: + tmpdir = Path(_tmpdir) + with chdir(tmpdir): + pyproject = tmpdir / "pyproject.toml" + write_config( + pyproject, + "[tool.mypy]\n" + 'extend = "./folder/mypy.ini"\n' + "strict = true\n" + "[[tool.mypy.overrides]]\n" + 'module = "c"\n' + 'enable_error_code = ["explicit-override"]\n' + "disallow_untyped_defs = true", + ) + folder = tmpdir / "folder" + folder.mkdir() + write_config( + folder / "mypy.ini", + "[mypy]\n" + "strict = False\n" + "ignore_missing_imports_per_module = True\n" + "[mypy-c]\n" + "disallow_incomplete_defs = True", + ) + + options = Options() + strict_option_set = False + + def set_strict_flags() -> None: + nonlocal strict_option_set + strict_option_set = True + + stdout = io.StringIO() + stderr = io.StringIO() + parse_config_file(options, set_strict_flags, None, stdout, stderr) + + assert strict_option_set is True + assert options.ignore_missing_imports_per_module is True + assert options.config_file == str(pyproject.name) + if os.path.realpath(pyproject.parent).startswith("/private"): + # MacOS has some odd symlinks for tmp folder, resolve them to get the actual values + expected_path = os.path.realpath(pyproject.parent) + else: + expected_path = str(pyproject.parent) + assert os.environ["MYPY_CONFIG_FILE_DIR"] == expected_path + + assert options.per_module_options["c"] == { + "disable_error_code": [], + "enable_error_code": ["explicit-override"], + "disallow_untyped_defs": True, + "disallow_incomplete_defs": True, + } + + assert stdout.getvalue() == "" + assert stderr.getvalue() == "" + + def test_extend_cyclic(self) -> None: + with tempfile.TemporaryDirectory() as _tmpdir: + tmpdir = Path(_tmpdir) + with chdir(tmpdir): + pyproject = tmpdir / "pyproject.toml" + write_config(pyproject, '[tool.mypy]\nextend = "./folder/mypy.ini"\n') + + folder = tmpdir / "folder" + folder.mkdir() + ini = folder / "mypy.ini" + write_config(ini, "[mypy]\nextend = ../pyproject.toml\n") + + options = Options() + + stdout = io.StringIO() + stderr = io.StringIO() + parse_config_file(options, lambda: None, None, stdout, stderr) + + if os.path.realpath(pyproject).startswith("/private"): + # MacOS has some odd symlinks for tmp folder, resolve them to get the actual values + expected_pyproject = os.path.realpath(pyproject) + expected_ini = os.path.realpath(ini) + else: + expected_pyproject = str(pyproject) + expected_ini = str(ini) + + assert stdout.getvalue() == "" + assert stderr.getvalue() == ( + f"Circular extend detected: {expected_pyproject}\n" + f"../pyproject.toml is not a valid path to extend from {expected_ini}\n" + ) + + def test_extend_strict_override(self) -> None: + with tempfile.TemporaryDirectory() as _tmpdir: + tmpdir = Path(_tmpdir) + with chdir(tmpdir): + pyproject = tmpdir / "pyproject.toml" + write_config( + pyproject, '[tool.mypy]\nextend = "./folder/mypy.ini"\nstrict = True\n' + ) + + folder = tmpdir / "folder" + folder.mkdir() + ini = folder / "mypy.ini" + write_config(ini, "[mypy]\nstrict = false\n") + + options = Options() + + stdout = io.StringIO() + stderr = io.StringIO() + + strict_called = False + + def set_strict_flags() -> None: + nonlocal strict_called + strict_called = True + + parse_config_file(options, set_strict_flags, None, stdout, stderr) + + assert strict_called is False