From 68d0c48619978862b9a5466acde755170ed1d672 Mon Sep 17 00:00:00 2001 From: Simon Brunning Date: Fri, 6 Aug 2021 13:13:09 +0100 Subject: [PATCH 01/11] Some refactoring before making the change - use an object for expected output rather than a string. --- .gitignore | 2 + pytest_mypy_plugins/collect.py | 15 ++- pytest_mypy_plugins/item.py | 7 +- .../tests/test-regex_assertions.yml | 9 ++ pytest_mypy_plugins/utils.py | 117 ++++++++++++++---- 5 files changed, 116 insertions(+), 34 deletions(-) create mode 100644 pytest_mypy_plugins/tests/test-regex_assertions.yml diff --git a/.gitignore b/.gitignore index 5ba8ffa..b7258a6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ __pycache__ dist/ build/ +.pytest_cache/ +venv/ diff --git a/pytest_mypy_plugins/collect.py b/pytest_mypy_plugins/collect.py index a527f71..7d33518 100644 --- a/pytest_mypy_plugins/collect.py +++ b/pytest_mypy_plugins/collect.py @@ -101,16 +101,21 @@ def collect(self) -> Iterator["YamlTestItem"]: test_name = f"{test_name_prefix}{test_name_suffix}" main_file = File(path="main.py", content=pystache.render(raw_test["main"], params)) test_files = [main_file] + parse_test_files(raw_test.get("files", [])) + regex = raw_test.get("regex", False) - output_from_comments = [] + expected_output = [] for test_file in test_files: - output_lines = utils.extract_errors_from_comments(test_file.path, test_file.content.split("\n")) - output_from_comments.extend(output_lines) + output_lines = utils.extract_output_matchers_from_comments( + test_file.path, test_file.content.split("\n"), regex=regex + ) + expected_output.extend(output_lines) starting_lineno = raw_test["__line__"] extra_environment_variables = parse_environment_variables(raw_test.get("env", [])) disable_cache = raw_test.get("disable_cache", False) - expected_output_lines = pystache.render(raw_test.get("out", ""), params).split("\n") + expected_output.extend( + utils.extract_output_matchers_from_out(raw_test.get("out", ""), params, regex=regex) + ) additional_mypy_config = raw_test.get("mypy_config", "") skip = self._eval_skip(str(raw_test.get("skip", "False"))) @@ -122,7 +127,7 @@ def collect(self) -> Iterator["YamlTestItem"]: starting_lineno=starting_lineno, environment_variables=extra_environment_variables, disable_cache=disable_cache, - expected_output_lines=output_from_comments + expected_output_lines, + expected_output=expected_output, parsed_test_data=raw_test, mypy_config=additional_mypy_config, ) diff --git a/pytest_mypy_plugins/item.py b/pytest_mypy_plugins/item.py index 775f731..c77dd6b 100644 --- a/pytest_mypy_plugins/item.py +++ b/pytest_mypy_plugins/item.py @@ -32,6 +32,7 @@ from pytest_mypy_plugins import utils from pytest_mypy_plugins.collect import File, YamlTestFile from pytest_mypy_plugins.utils import ( + OutputMatcher, TypecheckAssertionError, assert_string_arrays_equal, capture_std_streams, @@ -124,7 +125,7 @@ def __init__( *, files: List[File], starting_lineno: int, - expected_output_lines: List[str], + expected_output: List[OutputMatcher], environment_variables: Dict[str, Any], disable_cache: bool, mypy_config: str, @@ -134,7 +135,7 @@ def __init__( self.files = files self.environment_variables = environment_variables self.disable_cache = disable_cache - self.expected_output_lines = expected_output_lines + self.expected_output = expected_output self.starting_lineno = starting_lineno self.additional_mypy_config = mypy_config self.parsed_test_data = parsed_test_data @@ -279,7 +280,7 @@ def runtest(self) -> None: for line in mypy_output.splitlines(): output_line = replace_fpath_with_module_name(line, rootdir=execution_path) output_lines.append(output_line) - assert_string_arrays_equal(expected=self.expected_output_lines, actual=output_lines) + assert_string_arrays_equal(expected=self.expected_output, actual=output_lines) finally: temp_dir.cleanup() # remove created modules and all their dependants from cache diff --git a/pytest_mypy_plugins/tests/test-regex_assertions.yml b/pytest_mypy_plugins/tests/test-regex_assertions.yml new file mode 100644 index 0000000..f3618c1 --- /dev/null +++ b/pytest_mypy_plugins/tests/test-regex_assertions.yml @@ -0,0 +1,9 @@ +- case: expected_message_regex + skip: yes + regex: yes + main: | + a = 1 + b = 'hello' + + reveal_type(a) # N: Revealed type is "builtins.int" + reveal_type(b) # N: .*str.* diff --git a/pytest_mypy_plugins/utils.py b/pytest_mypy_plugins/utils.py index 40ec467..317a3a0 100644 --- a/pytest_mypy_plugins/utils.py +++ b/pytest_mypy_plugins/utils.py @@ -6,9 +6,20 @@ import os import re import sys +from dataclasses import dataclass from pathlib import Path -from typing import Callable, Iterator, List, Optional, Tuple, Union - +from typing import ( + Any, + Callable, + Iterator, + List, + Mapping, + Optional, + Tuple, + Union, +) + +import pystache from decorator import contextmanager @@ -55,6 +66,31 @@ def fname_to_module(fpath: Path, root_path: Path) -> Optional[str]: MIN_LINE_LENGTH_FOR_ALIGNMENT = 5 +@dataclass +class OutputMatcher: + fname: str + lnum: int + severity: str + message: str + regex: bool + col: Optional[str] = None + + def matches(self, actual: str) -> bool: + return str(self) == actual + + def __str__(self) -> str: + if self.col is None: + return f"{self.fname}:{self.lnum}: {self.severity}: {self.message}" + else: + return f"{self.fname}:{self.lnum}:{self.col}: {self.severity}: {self.message}" + + def __format__(self, format_spec: str) -> str: + return format_spec.format(str(self)) + + def __len__(self) -> int: + return len(str(self)) + + class TypecheckAssertionError(AssertionError): def __init__(self, error_message: Optional[str] = None, lineno: int = 0) -> None: self.error_message = error_message or "" @@ -81,16 +117,16 @@ def remove_common_prefix(lines: List[str]) -> List[str]: return cleaned_lines -def _num_skipped_prefix_lines(a1: List[str], a2: List[str]) -> int: +def _num_skipped_prefix_lines(a1: List[OutputMatcher], a2: List[str]) -> int: num_eq = 0 - while num_eq < min(len(a1), len(a2)) and a1[num_eq] == a2[num_eq]: + while num_eq < min(len(a1), len(a2)) and a1[num_eq].matches(a2[num_eq]): num_eq += 1 return max(0, num_eq - 4) -def _num_skipped_suffix_lines(a1: List[str], a2: List[str]) -> int: +def _num_skipped_suffix_lines(a1: List[OutputMatcher], a2: List[str]) -> int: num_eq = 0 - while num_eq < min(len(a1), len(a2)) and a1[-num_eq - 1] == a2[-num_eq - 1]: + while num_eq < min(len(a1), len(a2)) and a1[-num_eq - 1].matches(a2[-num_eq - 1]): num_eq += 1 return max(0, num_eq - 4) @@ -171,18 +207,23 @@ def extract_parts_as_tuple(line: str) -> Tuple[str, int, str]: return sorted(lines, key=extract_parts_as_tuple) -def assert_string_arrays_equal(expected: List[str], actual: List[str]) -> None: +def sorted_output_matchers_by_file_and_line(lines: List[OutputMatcher]) -> List[OutputMatcher]: + return sorted(lines, key=lambda om: (om.fname, om.lnum)) + + +def assert_string_arrays_equal(expected: List[OutputMatcher], actual: List[str]) -> None: """Assert that two string arrays are equal. Display any differences in a human-readable form. """ - expected = sorted_by_file_and_line(remove_empty_lines(expected)) + expected = sorted_output_matchers_by_file_and_line(expected) actual = sorted_by_file_and_line(remove_empty_lines(actual)) actual = remove_common_prefix(actual) error_message = "" - if expected != actual: + # TODO here! + if not all(e.matches(a) for e, a in zip(expected, actual)): num_skip_start = _num_skipped_prefix_lines(expected, actual) num_skip_end = _num_skipped_suffix_lines(expected, actual) @@ -200,13 +241,13 @@ def assert_string_arrays_equal(expected: List[str], actual: List[str]) -> None: width = 100 for i in range(num_skip_start, len(expected) - num_skip_end): - if i >= len(actual) or expected[i] != actual[i]: + if i >= len(actual) or not expected[i].matches(actual[i]): if first_diff < 0: first_diff = i error_message += " {:<45} (diff)".format(expected[i]) else: e = expected[i] - error_message += " " + e[:width] + error_message += " " + str(e)[:width] if len(e) > width: error_message += "..." error_message += "\n" @@ -219,7 +260,7 @@ def assert_string_arrays_equal(expected: List[str], actual: List[str]) -> None: error_message += " ...\n" for j in range(num_skip_start, len(actual) - num_skip_end): - if j >= len(expected) or expected[j] != actual[j]: + if j >= len(expected) or not expected[j].matches(actual[j]): error_message += " {:<45} (diff)".format(actual[j]) else: a = actual[j] @@ -240,33 +281,26 @@ def assert_string_arrays_equal(expected: List[str], actual: List[str]) -> None: ): # Display message that helps visualize the differences between two # long lines. - error_message = _add_aligned_message(expected[first_diff], actual[first_diff], error_message) + error_message = _add_aligned_message(str(expected[first_diff]), actual[first_diff], error_message) if len(expected) == 0: raise TypecheckAssertionError(f"Output is not expected: \n{error_message}") first_failure = expected[first_diff] if first_failure: - lineno = int(first_failure.split(" ")[0].strip(":").split(":")[1]) + lineno = first_failure.lnum raise TypecheckAssertionError(error_message=f"Invalid output: \n{error_message}", lineno=lineno) -def build_output_line(fname: str, lnum: int, severity: str, message: str, col: Optional[str] = None) -> str: - if col is None: - return f"{fname}:{lnum + 1}: {severity}: {message}" - else: - return f"{fname}:{lnum + 1}:{col}: {severity}: {message}" - - -def extract_errors_from_comments(fname: str, input_lines: List[str]) -> List[str]: +def extract_output_matchers_from_comments(fname: str, input_lines: List[str], regex: bool) -> List[OutputMatcher]: """Transform comments such as '# E: message' or '# E:3: message' in input. The result is lines like 'fnam:line: error: message'. """ fname = fname.replace(".py", "") - output_lines = [] - for lnum, line in enumerate(input_lines): + matchers = [] + for index, line in enumerate(input_lines): # The first in the split things isn't a comment for possible_err_comment in line.split(" # ")[1:]: m = re.search(r"^([ENW]):((?P\d+):)? (?P.*)$", possible_err_comment.strip()) @@ -278,8 +312,39 @@ def extract_errors_from_comments(fname: str, input_lines: List[str]) -> List[str elif m.group(1) == "W": severity = "warning" col = m.group("col") - output_lines.append(build_output_line(fname, lnum, severity, message=m.group("message"), col=col)) - return output_lines + matchers.append( + OutputMatcher(fname, index + 1, severity, message=m.group("message"), regex=regex, col=col) + ) + return matchers + + +def extract_output_matchers_from_out(out: str, params: Mapping[str, Any], regex: bool) -> List[OutputMatcher]: + matchers = [] + for line in pystache.render(out, params).split("\n"): + match = re.search( + r"^(?P.*):(?P\d*): (?P.*):((?P\d+):)? (?P.*)$", line.strip() + ) + if match: + if match.group("severity") == "E": + severity = "error" + elif match.group("severity") == "N": + severity = "note" + elif match.group("severity") == "W": + severity = "warning" + else: + severity = match.group("severity") + col = match.group("col") + matchers.append( + OutputMatcher( + match.group("fname"), + int(match.group("lnum")), + severity, + message=match.group("message"), + regex=regex, + col=col, + ) + ) + return matchers def get_func_first_lnum(attr: Callable[..., None]) -> Optional[Tuple[int, List[str]]]: From 77bc3a05ba326f923b4fb1f6572d55e6dc81bd61 Mon Sep 17 00:00:00 2001 From: Simon Brunning Date: Fri, 6 Aug 2021 13:55:01 +0100 Subject: [PATCH 02/11] Simple regex case now working --- .../tests/test-regex_assertions.yml | 1 - pytest_mypy_plugins/utils.py | 14 +++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pytest_mypy_plugins/tests/test-regex_assertions.yml b/pytest_mypy_plugins/tests/test-regex_assertions.yml index f3618c1..234146b 100644 --- a/pytest_mypy_plugins/tests/test-regex_assertions.yml +++ b/pytest_mypy_plugins/tests/test-regex_assertions.yml @@ -1,5 +1,4 @@ - case: expected_message_regex - skip: yes regex: yes main: | a = 1 diff --git a/pytest_mypy_plugins/utils.py b/pytest_mypy_plugins/utils.py index 317a3a0..4818421 100644 --- a/pytest_mypy_plugins/utils.py +++ b/pytest_mypy_plugins/utils.py @@ -20,6 +20,7 @@ ) import pystache +import regex from decorator import contextmanager @@ -76,7 +77,18 @@ class OutputMatcher: col: Optional[str] = None def matches(self, actual: str) -> bool: - return str(self) == actual + if self.regex: + pattern = ( + regex.escape( + f"{self.fname}:{self.lnum}: {self.severity}: " + if self.col is None + else f"{self.fname}:{self.lnum}:{self.col}: {self.severity}: " + ) + + self.message + ) + return regex.match(pattern, actual) + else: + return str(self) == actual def __str__(self) -> str: if self.col is None: From ced9be443afaa5c86a7fae9210969b8a174f3cd4 Mon Sep 17 00:00:00 2001 From: Simon Brunning Date: Fri, 6 Aug 2021 14:02:01 +0100 Subject: [PATCH 03/11] A spot of clean-up. --- pytest_mypy_plugins/item.py | 4 ++-- pytest_mypy_plugins/utils.py | 14 ++++---------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/pytest_mypy_plugins/item.py b/pytest_mypy_plugins/item.py index c77dd6b..c2d89fd 100644 --- a/pytest_mypy_plugins/item.py +++ b/pytest_mypy_plugins/item.py @@ -34,7 +34,7 @@ from pytest_mypy_plugins.utils import ( OutputMatcher, TypecheckAssertionError, - assert_string_arrays_equal, + assert_expected_matched_actual, capture_std_streams, fname_to_module, ) @@ -280,7 +280,7 @@ def runtest(self) -> None: for line in mypy_output.splitlines(): output_line = replace_fpath_with_module_name(line, rootdir=execution_path) output_lines.append(output_line) - assert_string_arrays_equal(expected=self.expected_output, actual=output_lines) + assert_expected_matched_actual(expected=self.expected_output, actual=output_lines) finally: temp_dir.cleanup() # remove created modules and all their dependants from cache diff --git a/pytest_mypy_plugins/utils.py b/pytest_mypy_plugins/utils.py index 4818421..28c522c 100644 --- a/pytest_mypy_plugins/utils.py +++ b/pytest_mypy_plugins/utils.py @@ -219,22 +219,17 @@ def extract_parts_as_tuple(line: str) -> Tuple[str, int, str]: return sorted(lines, key=extract_parts_as_tuple) -def sorted_output_matchers_by_file_and_line(lines: List[OutputMatcher]) -> List[OutputMatcher]: - return sorted(lines, key=lambda om: (om.fname, om.lnum)) - - -def assert_string_arrays_equal(expected: List[OutputMatcher], actual: List[str]) -> None: +def assert_expected_matched_actual(expected: List[OutputMatcher], actual: List[str]) -> None: """Assert that two string arrays are equal. Display any differences in a human-readable form. """ - expected = sorted_output_matchers_by_file_and_line(expected) + expected = sorted(expected, key=lambda om: (om.fname, om.lnum)) actual = sorted_by_file_and_line(remove_empty_lines(actual)) actual = remove_common_prefix(actual) error_message = "" - # TODO here! if not all(e.matches(a) for e, a in zip(expected, actual)): num_skip_start = _num_skipped_prefix_lines(expected, actual) num_skip_end = _num_skipped_suffix_lines(expected, actual) @@ -280,7 +275,7 @@ def assert_string_arrays_equal(expected: List[OutputMatcher], actual: List[str]) if len(a) > width: error_message += "..." error_message += "\n" - if actual == []: + if not actual: error_message += " (empty)\n" if num_skip_end > 0: error_message += " ...\n" @@ -300,8 +295,7 @@ def assert_string_arrays_equal(expected: List[OutputMatcher], actual: List[str]) first_failure = expected[first_diff] if first_failure: - lineno = first_failure.lnum - raise TypecheckAssertionError(error_message=f"Invalid output: \n{error_message}", lineno=lineno) + raise TypecheckAssertionError(error_message=f"Invalid output: \n{error_message}", lineno=first_failure.lnum) def extract_output_matchers_from_comments(fname: str, input_lines: List[str], regex: bool) -> List[OutputMatcher]: From 5c9be301904dfb3cb122720f5ee4221f84281615 Mon Sep 17 00:00:00 2001 From: Simon Brunning Date: Fri, 6 Aug 2021 14:16:35 +0100 Subject: [PATCH 04/11] Document regex option. --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 598e44e..d2e1b9a 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ On top of that, each case must comply to following types: | `env` | `Optional[Dict[str, str]]={}` | Environmental variables to be provided inside of test run | | `parametrized` | `Optional[List[Parameter]]=[]`\* | List of parameters, similar to [`@pytest.mark.parametrize`](https://docs.pytest.org/en/stable/parametrize.html) | | `skip` | `str` | Expression evaluated with following globals set: `sys`, `os`, `pytest` and `platform` | +| `regex` | `str` | Allow regular expressions in comments to be matched against actual output. | (*) Appendix to **pseudo** types used above: @@ -126,6 +127,17 @@ Implementation notes: main:1: note: Revealed type is 'builtins.str' ``` +#### 4. Regular expressions in expectations + +```yaml +- case: with_out + regex: yes + main: | + reveal_type('abc') + out: | + main:1: note: .*str.* +``` + ## Options ``` From 95f655237308d540c284edff6d2067caee8b53f4 Mon Sep 17 00:00:00 2001 From: Simon Brunning Date: Fri, 6 Aug 2021 15:06:34 +0100 Subject: [PATCH 05/11] Specify default value for regex flag. --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index d2e1b9a..24960eb 100644 --- a/README.md +++ b/README.md @@ -52,17 +52,17 @@ You can also specify `PYTHONPATH`, `MYPYPATH`, or any other environment variable In general each test case is just an element in an array written in a properly formatted `YAML` file. On top of that, each case must comply to following types: -| Property | Type | Description | -| --------------- | ------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------- | -| `case` | `str` | Name of the test case, complies to `[a-zA-Z0-9]` pattern | -| `main` | `str` | Portion of the code as if written in `.py` file | -| `files` | `Optional[List[File]]=[]`\* | List of extra files to simulate imports if needed | -| `disable_cache` | `Optional[bool]=False` | Set to `true` disables `mypy` caching | -| `mypy_config` | `Optional[Dict[str, Union[str, int, bool, float]]]={}` | Inline `mypy` configuration, passed directly to `mypy` as `--config-file` option | -| `env` | `Optional[Dict[str, str]]={}` | Environmental variables to be provided inside of test run | -| `parametrized` | `Optional[List[Parameter]]=[]`\* | List of parameters, similar to [`@pytest.mark.parametrize`](https://docs.pytest.org/en/stable/parametrize.html) | -| `skip` | `str` | Expression evaluated with following globals set: `sys`, `os`, `pytest` and `platform` | -| `regex` | `str` | Allow regular expressions in comments to be matched against actual output. | +| Property | Type | Description | +| --------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------- | +| `case` | `str` | Name of the test case, complies to `[a-zA-Z0-9]` pattern | +| `main` | `str` | Portion of the code as if written in `.py` file | +| `files` | `Optional[List[File]]=[]`\* | List of extra files to simulate imports if needed | +| `disable_cache` | `Optional[bool]=False` | Set to `true` disables `mypy` caching | +| `mypy_config` | `Optional[Dict[str, Union[str, int, bool, float]]]={}` | Inline `mypy` configuration, passed directly to `mypy` as `--config-file` option | +| `env` | `Optional[Dict[str, str]]={}` | Environmental variables to be provided inside of test run | +| `parametrized` | `Optional[List[Parameter]]=[]`\* | List of parameters, similar to [`@pytest.mark.parametrize`](https://docs.pytest.org/en/stable/parametrize.html) | +| `skip` | `str` | Expression evaluated with following globals set: `sys`, `os`, `pytest` and `platform` | +| `regex` | `str` | Allow regular expressions in comments to be matched against actual output. Defaults to "no", i.e. matches full text.| (*) Appendix to **pseudo** types used above: From 9811300cba4e25809af4ed6999788b22f1d769a2 Mon Sep 17 00:00:00 2001 From: Simon Brunning Date: Fri, 6 Aug 2021 15:20:35 +0100 Subject: [PATCH 06/11] Add test for regexes inb out section. --- README.md | 7 ++++--- pytest_mypy_plugins/tests/test-regex_assertions.yml | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 24960eb..3716d66 100644 --- a/README.md +++ b/README.md @@ -130,12 +130,13 @@ Implementation notes: #### 4. Regular expressions in expectations ```yaml -- case: with_out +- case: expected_message_regex_with_out regex: yes main: | - reveal_type('abc') + a = 'abc' + reveal_type(a) out: | - main:1: note: .*str.* + main:2: note: .*str.* ``` ## Options diff --git a/pytest_mypy_plugins/tests/test-regex_assertions.yml b/pytest_mypy_plugins/tests/test-regex_assertions.yml index 234146b..a357281 100644 --- a/pytest_mypy_plugins/tests/test-regex_assertions.yml +++ b/pytest_mypy_plugins/tests/test-regex_assertions.yml @@ -6,3 +6,11 @@ reveal_type(a) # N: Revealed type is "builtins.int" reveal_type(b) # N: .*str.* + +- case: expected_message_regex_with_out + regex: yes + main: | + a = 'abc' + reveal_type(a) + out: | + main:2: note: .*str.* From 2019cb1e1043d91a62f28482992e24e44f90d9a3 Mon Sep 17 00:00:00 2001 From: Simon Brunning Date: Mon, 9 Aug 2021 10:09:52 +0100 Subject: [PATCH 07/11] Add test to ensure that regexes ony run if the flag is switched on. --- .github/workflows/test.yml | 4 +++- .../tests/test-regex_assertions.shouldfail.yml | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52dc486..635e69f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,7 +26,9 @@ jobs: pip install -U pip setuptools wheel pip install -r dev-requirements.txt - name: Run tests - run: pytest + run: pytest --ignore-glob="*.shouldfail.yml" + - name: Run test with expected failures + run: pytest pytest_mypy_plugins/tests/*.shouldfail.yml 2>&1 | grep "2 failed" lint: runs-on: ubuntu-latest diff --git a/pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml b/pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml new file mode 100644 index 0000000..b9202e3 --- /dev/null +++ b/pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml @@ -0,0 +1,9 @@ +- case: rexex_but_not_turned_on + main: | + a = 'hello' + reveal_type(a) # N: .*str.* + +- case: rexex_but_turned_off + main: | + a = 'hello' + reveal_type(a) # N: .*str.* From 8defa6af68b09a1b87d4d7f0efecd2b7d0588429 Mon Sep 17 00:00:00 2001 From: Simon Brunning Date: Mon, 9 Aug 2021 10:53:27 +0100 Subject: [PATCH 08/11] Allow regexes on specific messages. --- README.md | 9 ++++++ .../tests/test-regex_assertions.yml | 6 ++++ pytest_mypy_plugins/utils.py | 29 ++++++++++++++----- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3716d66..a370c12 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,15 @@ Implementation notes: main:2: note: .*str.* ``` +#### 5. Regular expressions specific lines of output. + +```yaml +- case: expected_single_message_regex + main: | + a = 'hello' + reveal_type(a) # NR: .*str.* +``` + ## Options ``` diff --git a/pytest_mypy_plugins/tests/test-regex_assertions.yml b/pytest_mypy_plugins/tests/test-regex_assertions.yml index a357281..0f515ec 100644 --- a/pytest_mypy_plugins/tests/test-regex_assertions.yml +++ b/pytest_mypy_plugins/tests/test-regex_assertions.yml @@ -14,3 +14,9 @@ reveal_type(a) out: | main:2: note: .*str.* + +- case: expected_single_message_regex + regex: no + main: | + a = 'hello' + reveal_type(a) # NR: .*str.* \ No newline at end of file diff --git a/pytest_mypy_plugins/utils.py b/pytest_mypy_plugins/utils.py index 28c522c..5c497f5 100644 --- a/pytest_mypy_plugins/utils.py +++ b/pytest_mypy_plugins/utils.py @@ -302,29 +302,42 @@ def extract_output_matchers_from_comments(fname: str, input_lines: List[str], re """Transform comments such as '# E: message' or '# E:3: message' in input. - The result is lines like 'fnam:line: error: message'. + The result is a list pf output matchers """ fname = fname.replace(".py", "") matchers = [] for index, line in enumerate(input_lines): # The first in the split things isn't a comment for possible_err_comment in line.split(" # ")[1:]: - m = re.search(r"^([ENW]):((?P\d+):)? (?P.*)$", possible_err_comment.strip()) - if m: - if m.group(1) == "E": + match = re.search(r"^([ENW])(?P[R]):((?P\d+):)? (?P.*)$", possible_err_comment.strip()) + if match: + if match.group(1) == "E": severity = "error" - elif m.group(1) == "N": + elif match.group(1) == "N": severity = "note" - elif m.group(1) == "W": + elif match.group(1) == "W": severity = "warning" - col = m.group("col") + else: + severity = match.group(1) + col = match.group("col") matchers.append( - OutputMatcher(fname, index + 1, severity, message=m.group("message"), regex=regex, col=col) + OutputMatcher( + fname, + index + 1, + severity, + message=match.group("message"), + regex=regex or bool(match.group("regex")), + col=col, + ) ) return matchers def extract_output_matchers_from_out(out: str, params: Mapping[str, Any], regex: bool) -> List[OutputMatcher]: + """Transform output lines such as 'function:9: E: message' + + The result is a list of output matchers + """ matchers = [] for line in pystache.render(out, params).split("\n"): match = re.search( From 848282e1211e5be9b8555e1bad3f52930fc5b98d Mon Sep 17 00:00:00 2001 From: Simon Brunning Date: Mon, 9 Aug 2021 11:18:16 +0100 Subject: [PATCH 09/11] Fix regex - single line flag froup needed to be optional. --- .../tests/test-regex_assertions.shouldfail.yml | 1 + pytest_mypy_plugins/tests/test-regex_assertions.yml | 2 +- pytest_mypy_plugins/utils.py | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml b/pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml index b9202e3..5c16f5d 100644 --- a/pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml +++ b/pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml @@ -4,6 +4,7 @@ reveal_type(a) # N: .*str.* - case: rexex_but_turned_off + regex: no main: | a = 'hello' reveal_type(a) # N: .*str.* diff --git a/pytest_mypy_plugins/tests/test-regex_assertions.yml b/pytest_mypy_plugins/tests/test-regex_assertions.yml index 0f515ec..0cdf230 100644 --- a/pytest_mypy_plugins/tests/test-regex_assertions.yml +++ b/pytest_mypy_plugins/tests/test-regex_assertions.yml @@ -19,4 +19,4 @@ regex: no main: | a = 'hello' - reveal_type(a) # NR: .*str.* \ No newline at end of file + reveal_type(a) # NR: .*str.* diff --git a/pytest_mypy_plugins/utils.py b/pytest_mypy_plugins/utils.py index 5c497f5..2e87e2c 100644 --- a/pytest_mypy_plugins/utils.py +++ b/pytest_mypy_plugins/utils.py @@ -309,7 +309,9 @@ def extract_output_matchers_from_comments(fname: str, input_lines: List[str], re for index, line in enumerate(input_lines): # The first in the split things isn't a comment for possible_err_comment in line.split(" # ")[1:]: - match = re.search(r"^([ENW])(?P[R]):((?P\d+):)? (?P.*)$", possible_err_comment.strip()) + match = re.search( + r"^([ENW])(?P[R])?:((?P\d+):)? (?P.*)$", possible_err_comment.strip() + ) if match: if match.group(1) == "E": severity = "error" From e814815e30727f6d74a0dc4d027ed6b4ccc80cda Mon Sep 17 00:00:00 2001 From: Simon Brunning Date: Mon, 9 Aug 2021 12:26:44 +0100 Subject: [PATCH 10/11] Add simple failing simple cases. --- .github/workflows/test.yml | 2 +- .../tests/test-simple-cases.shouldfail.yml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 pytest_mypy_plugins/tests/test-simple-cases.shouldfail.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 635e69f..baf4fe5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: - name: Run tests run: pytest --ignore-glob="*.shouldfail.yml" - name: Run test with expected failures - run: pytest pytest_mypy_plugins/tests/*.shouldfail.yml 2>&1 | grep "2 failed" + run: pytest pytest_mypy_plugins/tests/*.shouldfail.yml 2>&1 | grep "4 failed" lint: runs-on: ubuntu-latest diff --git a/pytest_mypy_plugins/tests/test-simple-cases.shouldfail.yml b/pytest_mypy_plugins/tests/test-simple-cases.shouldfail.yml new file mode 100644 index 0000000..032e96b --- /dev/null +++ b/pytest_mypy_plugins/tests/test-simple-cases.shouldfail.yml @@ -0,0 +1,12 @@ +- case: fail_if_message_does_not_match + main: | + a = 'hello' + reveal_type(a) # N: Some other message + +- case: fail_if_message_from_outdoes_not_match + regex: yes + main: | + a = 'abc' + reveal_type(a) + out: | + main:2: note: Some other message \ No newline at end of file From b6e684c5d131ee9675092484376c05b550ca8a27 Mon Sep 17 00:00:00 2001 From: Simon Brunning Date: Mon, 9 Aug 2021 12:28:34 +0100 Subject: [PATCH 11/11] Add case for mismatching regex. --- .github/workflows/test.yml | 2 +- .../tests/test-regex_assertions.shouldfail.yml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index baf4fe5..9dcdece 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: - name: Run tests run: pytest --ignore-glob="*.shouldfail.yml" - name: Run test with expected failures - run: pytest pytest_mypy_plugins/tests/*.shouldfail.yml 2>&1 | grep "4 failed" + run: pytest pytest_mypy_plugins/tests/*.shouldfail.yml 2>&1 | grep "5 failed" lint: runs-on: ubuntu-latest diff --git a/pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml b/pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml index 5c16f5d..7d064ea 100644 --- a/pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml +++ b/pytest_mypy_plugins/tests/test-regex_assertions.shouldfail.yml @@ -8,3 +8,9 @@ main: | a = 'hello' reveal_type(a) # N: .*str.* + +- case: regext_does_not_match + regex: no + main: | + a = 'hello' + reveal_type(a) # NR: .*banana.*