diff --git a/README.rst b/README.rst index c044a53..494985b 100644 --- a/README.rst +++ b/README.rst @@ -105,6 +105,39 @@ If you need to use any flag inside the tests, you can modify the pl.show() +Specifying multiple cache images +================================ +The cache directory is typically flat with no sub-directories. However, +it is possible to specify multiple cache images for a single test by +including a sub-directory with the same name as the test, and including +multiple "valid" cache images in the sub-directory. For example, a +single cached image: + +.. code-block:: bash + + cache/my_image.jpg + +Can be replaced with multiple versions of the image: + +.. code-block:: bash + + cache/my_image/0.jpg + cache/my_image/1.jpg + +.. note:: + + - The sub-directory name should match the name of the test. + - The image names in sub-directories can be arbitrary, e.g. ``0.jpg`` or + ``foo.jpg``. + - Nested sub-directories are also supported, and their names can also be arbitrary. + - Use the ``--generate_subdirs`` flag to automatically generate test images in a + sub-directory format. + +When there are multiple images, the test will initially compare the build image +to the first cached image. If that comparison fails, the build image is then +compared to all other cached images for that test. The test is successful if one +of the comparisons is successful, though a warning is still issued. + Global flags ------------ These are the flags you can use when calling ``pytest`` in the command line: @@ -120,6 +153,14 @@ These are the flags you can use when calling ``pytest`` in the command line: directory, relative to `pytest root path `. This will override any configuration, see below. +* ``--generate_subdirs`` saves generated test images in separate sub-directories + instead of saving them directly to the ``generated_image_dir``. Without this option, + generated images are saved as ``generated_image_dir/.png``; with this + option enabled, they are instead saved as + ``//.png``, where the image name has + the format ``___``. This can + be useful for providing context about how an image was generated. + * ``--failed_image_dir `` dumps copies of cached and generated test images when there is a warning or error raised. This directory is useful for reviewing test failures. It is relative to `pytest root path `. diff --git a/pytest_pyvista/pytest_pyvista.py b/pytest_pyvista/pytest_pyvista.py index eb1af2f..4995145 100644 --- a/pytest_pyvista/pytest_pyvista.py +++ b/pytest_pyvista/pytest_pyvista.py @@ -8,6 +8,7 @@ from pathlib import Path import platform import shutil +import sys from typing import TYPE_CHECKING from typing import Callable from typing import Literal @@ -18,6 +19,7 @@ import pytest import pyvista from pyvista import Plotter +import vtkmodules if TYPE_CHECKING: # pragma: no cover from collections.abc import Generator @@ -26,6 +28,24 @@ SKIPPED_CACHED_IMAGE_NAMES: set[str] = set() +def _get_env_info() -> str: + system = platform.system() + if system == "Darwin": + system = "macOS" + + return "_".join( + [ + f"{system}-{platform.release()}", + f"py-{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", + f"pyvista-{pyvista.__version__}", + f"vtk-{vtkmodules.__version__}", + ] + ) + + +ENV_INFO = _get_env_info() + + class RegressionError(RuntimeError): """Error when regression does not meet the criteria.""" @@ -67,6 +87,11 @@ def pytest_addoption(parser) -> None: # noqa: ANN001 default=None, help="Path to dump test images from the current run.", ) + group.addoption( + "--generate_subdirs", + action="store_true", + help="Save generated images to sub-directories.", + ) group.addoption( "--failed_image_dir", action="store", @@ -123,7 +148,7 @@ class VerifyImageCache: ---------- test_name : str Name of test to save. It is used to define the name of image cache - file. + file or sub-directory. cache_dir : Path Directory for image cache comparisons. @@ -171,6 +196,7 @@ class VerifyImageCache: allow_unused_generated = False add_missing_images = False reset_only_failed = False + generate_subdirs = None def __init__( # noqa: PLR0913 self, @@ -216,7 +242,7 @@ def _is_skipped(*, skip: bool, windows_skip_image_cache: bool, macos_skip_image_ skip_macos = platform.system() == "Darwin" and macos_skip_image_cache return skip or ignore_image_cache or skip_windows or skip_macos - def __call__(self, plotter: Plotter) -> None: # noqa: C901, PLR0912 + def __call__(self, plotter: Plotter) -> None: # noqa: C901, PLR0912, PLR0915 """ Either store or validate an image. @@ -246,7 +272,10 @@ def remove_plotter_close_callback() -> None: # "test_" to get the name for the image. image_name = _image_name_from_test_name(test_name) image_filename = Path(self.cache_dir, image_name) - gen_image_filename = None if self.generated_image_dir is None else Path(self.generated_image_dir, image_name) + image_dirname = Path(self.cache_dir, Path(image_name).stem) + + cached_image_paths = _get_file_paths(image_dirname, ext="png") if image_dirname.is_dir() else [image_filename] + current_cached_image = cached_image_paths[0] if VerifyImageCache._is_skipped( skip=self.skip, @@ -258,80 +287,118 @@ def remove_plotter_close_callback() -> None: return VISITED_CACHED_IMAGE_NAMES.add(image_name) - if not image_filename.is_file() and not (self.allow_unused_generated or self.add_missing_images or self.reset_image_cache): + if not current_cached_image.is_file() and not (self.allow_unused_generated or self.add_missing_images or self.reset_image_cache): # Raise error since the cached image does not exist and will not be added later # Save images as needed before error - if gen_image_filename is not None: - plotter.screenshot(gen_image_filename) + if self.generated_image_dir is not None: + self._save_generated_image(plotter, image_name=image_name) if self.failed_image_dir is not None: self._save_failed_test_images("error", plotter, image_name) remove_plotter_close_callback() - msg = f"{image_filename} does not exist in image cache" + msg = f"{current_cached_image} does not exist in image cache" raise RegressionFileNotFoundError(msg) - if (self.add_missing_images and not image_filename.is_file()) or (self.reset_image_cache and not self.reset_only_failed): - plotter.screenshot(image_filename) + if (self.add_missing_images and not current_cached_image.is_file()) or (self.reset_image_cache and not self.reset_only_failed): + plotter.screenshot(current_cached_image) - if gen_image_filename is not None: - plotter.screenshot(gen_image_filename) + if self.generated_image_dir is not None: + self._save_generated_image(plotter, image_name=image_name) - if not Path(image_filename).is_file() and self.allow_unused_generated: + if not Path(current_cached_image).is_file() and self.allow_unused_generated: # Test image has been generated, but cached image does not exist # The generated image is considered unused, so exit safely before image # comparison to avoid a FileNotFoundError return - if self.failed_image_dir is not None and not Path(image_filename).is_file(): - # Image comparison will fail, so save image before error - self._save_failed_test_images("error", plotter, image_name) - remove_plotter_close_callback() - - error = pyvista.compare_images(str(image_filename), plotter) - - if error > allowed_error: + test_name_no_prefix = test_name.removeprefix("test_") + warn_msg, fail_msg = _test_compare_images( + test_name=test_name_no_prefix, + test_image=plotter, + cached_image=str(current_cached_image), + allowed_error=allowed_error, + allowed_warning=allowed_warning, + ) + + # Try again and compare with other cached images + if fail_msg and len(cached_image_paths) > 1: + # Compare build image to other known valid versions + msg_start = "This test has multiple cached images. It initially failed (as above)" + for path in cached_image_paths: + error = pyvista.compare_images(plotter, str(path)) + if _check_compare_fail(test_name, error, allowed_error=allowed_error) is None: + # Convert failure into a warning + warn_msg = fail_msg + (f"\n{msg_start} but passed when compared to:\n\t{path}") + fail_msg = None + current_cached_image = path + break + else: # Loop completed - test still fails + fail_msg += f"\n{msg_start} and failed again for all images in:\n\t{Path(self.cache_dir, test_name_no_prefix)!s}" + + if fail_msg: if self.failed_image_dir is not None: self._save_failed_test_images("error", plotter, image_name) if self.reset_only_failed: warnings.warn( - f"{test_name} Exceeded image regression error of " - f"{allowed_error} with an image error equal to: {error}" - f"\nThis image will be reset in the cache.", + f"{fail_msg}\nThis image will be reset in the cache.", stacklevel=2, ) - plotter.screenshot(image_filename) + plotter.screenshot(current_cached_image) else: remove_plotter_close_callback() - msg = f"{test_name} Exceeded image regression error of {allowed_error} with an image error equal to: {error}" - raise RegressionError(msg) - if error > allowed_warning: + raise RegressionError(fail_msg) + + if warn_msg: + parent_dir: Literal["errors_as_warning", "warning"] = "errors_as_warning" if image_dirname.is_dir() else "warning" if self.failed_image_dir is not None: - self._save_failed_test_images("warning", plotter, image_name) - warnings.warn(f"{test_name} Exceeded image regression warning of {allowed_warning} with an image error of {error}", stacklevel=2) + self._save_failed_test_images(parent_dir, plotter, image_name, cache_image_path=current_cached_image) + warnings.warn(warn_msg, stacklevel=2) - def _save_failed_test_images(self, error_or_warning: Literal["error", "warning"], plotter: Plotter, image_name: str) -> None: + def _save_generated_image(self, plotter: pyvista.Plotter, image_name: str, parent_dir: Path | None = None) -> None: + parent = cast("Path", self.generated_image_dir) if parent_dir is None else parent_dir + generated_image_path = parent / Path(image_name).with_suffix("") / (ENV_INFO + ".png") if self.generate_subdirs else parent / image_name + generated_image_path.parent.mkdir(exist_ok=True, parents=True) + plotter.screenshot(generated_image_path) + + def _save_failed_test_images( + self, + error_or_warning: Literal["error", "warning", "errors_as_warning"], + plotter: Plotter, + image_name: str, + cache_image_path: Path | None = None, + ) -> None: """Save test image from cache and from test to the failed image dir.""" def _make_failed_test_image_dir( - errors_or_warnings: Literal["errors", "warnings"], from_cache_or_test: Literal["from_cache", "from_test"] + errors_or_warnings: Literal["errors", "warnings", "errors_as_warnings"], from_cache_or_test: Literal["from_cache", "from_test"] ) -> Path: # Check was done earlier to verify this is not None failed_image_dir = cast("str", self.failed_image_dir) - _ensure_dir_exists(failed_image_dir, msg_name="failed image dir") dest_dir = Path(failed_image_dir, errors_or_warnings, from_cache_or_test) dest_dir.mkdir(exist_ok=True, parents=True) return dest_dir - error_dirname = cast("Literal['errors', 'warnings']", error_or_warning + "s") + def _save_single_cache_image(path: Path) -> None: + rel = Path(path).relative_to(self.cache_dir) + from_cache_dir = _make_failed_test_image_dir(error_dirname, "from_cache") + dest_path = from_cache_dir / rel + dest_path.parent.mkdir(exist_ok=True, parents=True) + shutil.copy(path, dest_path) + + error_dirname = cast("Literal['errors', 'warnings', 'errors_as_warnings']", error_or_warning + "s") from_test_dir = _make_failed_test_image_dir(error_dirname, "from_test") - plotter.screenshot(from_test_dir / image_name) + self._save_generated_image(plotter, image_name=image_name, parent_dir=from_test_dir) - cached_image = Path(self.cache_dir, image_name) + cached_image = Path(self.cache_dir, image_name) if cache_image_path is None else cache_image_path if cached_image.is_file(): - from_cache_dir = _make_failed_test_image_dir(error_dirname, "from_cache") - shutil.copy(cached_image, from_cache_dir / image_name) + # Save single cache file + _save_single_cache_image(cached_image) + elif (image_dir := cached_image.with_suffix("")).is_dir(): + # Save multiple cached files + for path in _get_file_paths(image_dir, ext="png"): + _save_single_cache_image(path) def _image_name_from_test_name(test_name: str) -> str: @@ -354,6 +421,33 @@ def remove_suffix(s: str) -> str: return "test_" + remove_suffix(image_name) +def _get_file_paths(dir_: Path, ext: str) -> list[Path]: + """Get all paths of files with a specific extension inside a directory tree.""" + return sorted(dir_.rglob(f"*.{ext}")) + + +def _test_compare_images( + test_name: str, test_image: str | pyvista.Plotter, cached_image: str | pyvista.Plotter, allowed_error: float, allowed_warning: float +) -> tuple[str | None, str | None]: + # Check if test should fail or warn + error = pyvista.compare_images(test_image, cached_image) + fail_msg = _check_compare_fail(test_name, error, allowed_error) + warn_msg = _check_compare_warn(test_name, error, allowed_warning) + return warn_msg, fail_msg + + +def _check_compare_fail(test_name: str, error_: float, allowed_error: float) -> str | None: + if error_ > allowed_error: + return f"{test_name} Exceeded image regression error of {allowed_error} with an image error equal to: {error_}" + return None + + +def _check_compare_warn(test_name: str, error_: float, allowed_warning: float) -> str | None: + if error_ > allowed_warning: + return f"{test_name} Exceeded image regression warning of {allowed_warning} with an image error of {error_}" + return None + + @pytest.hookimpl def pytest_terminal_summary(terminalreporter, exitstatus, config) -> None: # noqa: ANN001, ARG001 """Execute after the whole test run completes.""" @@ -471,6 +565,7 @@ def verify_image_cache( VerifyImageCache.allow_unused_generated = pytestconfig.getoption("allow_unused_generated") VerifyImageCache.add_missing_images = pytestconfig.getoption("add_missing_images") VerifyImageCache.reset_only_failed = pytestconfig.getoption("reset_only_failed") + VerifyImageCache.generate_subdirs = pytestconfig.getoption("generate_subdirs") cache_dir = _get_option_from_config_or_ini(pytestconfig, "image_cache_dir", is_dir=True) gen_dir = _get_option_from_config_or_ini(pytestconfig, "generated_image_dir", is_dir=True) diff --git a/tests/test_pyvista.py b/tests/test_pyvista.py index 1e31e08..b62f455 100644 --- a/tests/test_pyvista.py +++ b/tests/test_pyvista.py @@ -12,6 +12,8 @@ import pytest import pyvista as pv +from pytest_pyvista.pytest_pyvista import _get_env_info + pv.OFF_SCREEN = True pytest_plugins = "pytester" @@ -35,7 +37,7 @@ def test_args(verify_image_cache): def make_cached_images(test_path, path="image_cache_dir", name="imcache.png", color="red") -> Path: """Make image cache in `test_path/path`.""" d = Path(test_path, path) - d.mkdir(exist_ok=True) + d.mkdir(exist_ok=True, parents=True) sphere = pv.Sphere() plotter = pv.Plotter() plotter.add_mesh(sphere, color=color) @@ -284,7 +286,8 @@ def test_imcache_var(verify_image_cache): result.assert_outcomes(passed=1) -def test_generated_image_dir_commandline(pytester: pytest.Pytester) -> None: +@pytest.mark.parametrize("generate_subdirs", [True, False]) +def test_generated_image_dir_commandline(pytester: pytest.Pytester, generate_subdirs) -> None: """Test setting generated_image_dir via CLI option.""" make_cached_images(pytester.path) pytester.makepyfile( @@ -298,10 +301,17 @@ def test_imcache(verify_image_cache): plotter.show() """ ) + args = ["--generated_image_dir", "gen_dir"] + if generate_subdirs: + args.append("--generate_subdirs") - result = pytester.runpytest("--generated_image_dir", "gen_dir") + result = pytester.runpytest(*args) assert (pytester.path / "gen_dir").is_dir() - assert (pytester.path / "gen_dir" / "imcache.png").is_file() + if generate_subdirs: + with_suffix = _get_env_info() + ".png" + assert (pytester.path / "gen_dir" / "imcache" / with_suffix).is_file() + else: + assert (pytester.path / "gen_dir" / "imcache.png").is_file() result.assert_outcomes(passed=1) @@ -566,7 +576,6 @@ def test_imcache(verify_image_cache): if outcome == "success": assert not failed_image_dir_path.is_dir() else: - result.stdout.fnmatch_lines("*UserWarning: pyvista test failed image dir: *failed_image_dir does not yet exist. Creating dir.") if make_cache: result.stdout.fnmatch_lines(f"*Exceeded image regression {outcome}*") else: @@ -881,3 +890,95 @@ def test_auto_close_2(verify_image_cache, mocker): result = pytester.runpytest() result.assert_outcomes(passed=2) + + +ALMOST_BLUE = [0, 0, 254] +ALMOST_RED = [254, 0, 0] + + +@pytest.mark.parametrize("failed_image_dir", [True, False]) +@pytest.mark.parametrize("nested_subdir", [True, False]) +@pytest.mark.parametrize( + ("build_color", "return_code"), [(ALMOST_RED, pytest.ExitCode.OK), (ALMOST_BLUE, pytest.ExitCode.OK), ("'green'", pytest.ExitCode.TESTS_FAILED)] +) +def test_multiple_cache_images(pytester: pytest.Pytester, build_color, return_code, nested_subdir, failed_image_dir) -> None: + """Test regression warning is issued.""" + cache = "cache" + name = "imcache.png" + subdir = Path(name).stem + cache_parent = pytester.path / cache + cache_parent = cache_parent / subdir if nested_subdir else cache_parent + red_filename = make_cached_images(cache_parent, subdir, name="im1.png", color="red") + blue_filename = make_cached_images(cache_parent, subdir, name="im2.png", color="blue") + + pyfile = f""" + import pyvista as pv + pv.OFF_SCREEN = True + def test_imcache(verify_image_cache): + sphere = pv.Sphere() + plotter = pv.Plotter() + plotter.add_mesh(sphere, color={build_color}) + plotter.show() + """ + pytester.makepyfile(pyfile) + + args = ["--image_cache_dir", cache] + failed = "failed" + if failed_image_dir: + args.extend(["--failed_image_dir", failed]) + result = pytester.runpytest(*args) + assert result.ret == return_code + + partial_match = r"imcache Exceeded image regression error of 500\.0 with an image error equal to: [0-9]+\.[0-9]+" + rel_subdirs = Path(subdir) / subdir if nested_subdir else subdir + + if build_color == ALMOST_RED: + # Comparison with first image succeeds without issue + result.stdout.no_re_match_line(rf".*UserWarning: {partial_match}") + + # Test no images are saved + assert not Path(failed).is_dir() + + elif build_color == ALMOST_BLUE: + # Comparison with first image fails + # Expect error was converted to a warning + result.stdout.re_match_lines( + [ + rf".*UserWarning: {partial_match}", + r".*This test has multiple cached images. It initially failed \(as above\) but passed when compared to:", + ".*im2.png", + ] + ) + # Test failed images are saved + from_cache = Path(failed) / "errors_as_warnings" / "from_cache" / rel_subdirs / blue_filename.name + assert from_cache.is_file() == failed_image_dir + if failed_image_dir: + assert not file_has_changed(str(from_cache), str(blue_filename)) + + from_test = Path(failed, "errors_as_warnings", "from_test", name) + assert from_test.is_file() == failed_image_dir + if failed_image_dir: + assert file_has_changed(str(from_test), str(from_cache)) + + else: # 'green' + # Comparison with all cached images fails + result.stdout.re_match_lines( + [ + rf".*RegressionError: {partial_match}", + r".*This test has multiple cached images. It initially failed \(as above\) and failed again for all images in:", + ".*cache/imcache", + ] + ) + + # Test failed images are saved + # Expect both red and blue cached images saved + for filename in [blue_filename, red_filename]: + from_cache = Path(failed) / "errors" / "from_cache" / rel_subdirs / filename.name + assert from_cache.is_file() == failed_image_dir + if failed_image_dir: + assert not file_has_changed(str(from_cache), str(filename)) + + from_test = Path(failed, "errors", "from_test", name) + assert from_test.is_file() == failed_image_dir + if failed_image_dir: + assert file_has_changed(str(from_test), str(from_cache))