Allow using multiple cache images

README.rst

@@ -105,6 +105,39 @@ If you need to use any flag inside the tests, you can modify the
         pl.show()
 
 
+Specifying multiple cache images
+================================
+The cache directory is typically flat with no sub-directories. However,
+it is possible to specify multiple cache images for a single test by
+including a sub-directory with the same name as the test, and including
+multiple "valid" cache images in the sub-directory. For example, a
+single cached image:
+
+.. code-block:: bash
+
+    cache/my_image.jpg
+
+Can be replaced with multiple versions of the image:
+
+.. code-block:: bash
+
+    cache/my_image/0.jpg
+    cache/my_image/1.jpg
+
+.. note::
+
+   - The sub-directory name should match the name of the test.
+   - The image names in sub-directories can be arbitrary, e.g. ``0.jpg`` or
+     ``foo.jpg``.
+   - Nested sub-directories are also supported, and their names can also be arbitrary.
+   - Use the ``--generate_subdirs`` flag to automatically generate test images in a
+     sub-directory format.
+
+When there are multiple images, the test will initially compare the build image
+to the first cached image. If that comparison fails, the build image is then
+compared to all other cached images for that test. The test is successful if one
+of the comparisons is successful, though a warning is still issued.
+
 Global flags
 ------------
 These are the flags you can use when calling ``pytest`` in the command line:
@@ -120,6 +153,14 @@ These are the flags you can use when calling ``pytest`` in the command line:
   directory, relative to `pytest root path <https://docs.pytest.org/en/latest/reference/reference.html#pytest.Config.rootpath>`.
   This will override any configuration, see below.
 
+* ``--generate_subdirs`` saves generated test images in separate sub-directories
+  instead of saving them directly to the ``generated_image_dir``. Without this option,
+  generated images are saved as ``generated_image_dir/<test_name>.png``; with this
+  option enabled, they are instead saved as
+  ``<generated_image_dir>/<test_name>/<image_name>.png``, where the image name has
+  the format ``<system>_<python-version>_<pyvista-version>_<vtk-version>``. This can
+  be useful for providing context about how an image was generated.
+
 * ``--failed_image_dir <DIR>`` dumps copies of cached and generated test images when
   there is a warning or error raised. This directory is useful for reviewing test
   failures. It is relative to `pytest root path <https://docs.pytest.org/en/latest/reference/reference.html#pytest.Config.rootpath>`.

pytest_pyvista/pytest_pyvista.py

@@ -8,6 +8,7 @@
 from pathlib import Path
 import platform
 import shutil
+import sys
 from typing import TYPE_CHECKING
 from typing import Callable
 from typing import Literal
@@ -18,6 +19,7 @@
 import pytest
 import pyvista
 from pyvista import Plotter
+import vtkmodules
 
 if TYPE_CHECKING:  # pragma: no cover
     from collections.abc import Generator
@@ -26,6 +28,24 @@
 SKIPPED_CACHED_IMAGE_NAMES: set[str] = set()
 
 
+def _get_env_info() -> str:
+    system = platform.system()
+    if system == "Darwin":
+        system = "macOS"
+
+    return "_".join(
+        [
+            f"{system}-{platform.release()}",
+            f"py-{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}",
+            f"pyvista-{pyvista.__version__}",
+            f"vtk-{vtkmodules.__version__}",
+        ]
+    )
+
+
+ENV_INFO = _get_env_info()
+
+
 class RegressionError(RuntimeError):
     """Error when regression does not meet the criteria."""
 
@@ -67,6 +87,11 @@ def pytest_addoption(parser) -> None:  # noqa: ANN001
         default=None,
         help="Path to dump test images from the current run.",
     )
+    group.addoption(
+        "--generate_subdirs",
+        action="store_true",
+        help="Save generated images to sub-directories.",
+    )
     group.addoption(
         "--failed_image_dir",
         action="store",
@@ -123,7 +148,7 @@ class VerifyImageCache:
     ----------
     test_name : str
         Name of test to save.  It is used to define the name of image cache
-        file.
+        file or sub-directory.
 
     cache_dir : Path
         Directory for image cache comparisons.
@@ -171,6 +196,7 @@ class VerifyImageCache:
     allow_unused_generated = False
     add_missing_images = False
     reset_only_failed = False
+    generate_subdirs = None
 
     def __init__(  # noqa: PLR0913
         self,
@@ -216,7 +242,7 @@ def _is_skipped(*, skip: bool, windows_skip_image_cache: bool, macos_skip_image_
         skip_macos = platform.system() == "Darwin" and macos_skip_image_cache
         return skip or ignore_image_cache or skip_windows or skip_macos
 
-    def __call__(self, plotter: Plotter) -> None:  # noqa: C901, PLR0912
+    def __call__(self, plotter: Plotter) -> None:  # noqa: C901, PLR0912, PLR0915
         """
         Either store or validate an image.
 
@@ -246,7 +272,10 @@ def remove_plotter_close_callback() -> None:
         # "test_" to get the name for the image.
         image_name = _image_name_from_test_name(test_name)
         image_filename = Path(self.cache_dir, image_name)
-        gen_image_filename = None if self.generated_image_dir is None else Path(self.generated_image_dir, image_name)
+        image_dirname = Path(self.cache_dir, Path(image_name).stem)
+
+        cached_image_paths = _get_file_paths(image_dirname, ext="png") if image_dirname.is_dir() else [image_filename]
+        current_cached_image = cached_image_paths[0]
 
         if VerifyImageCache._is_skipped(
             skip=self.skip,
@@ -258,80 +287,118 @@ def remove_plotter_close_callback() -> None:
             return
         VISITED_CACHED_IMAGE_NAMES.add(image_name)
 
-        if not image_filename.is_file() and not (self.allow_unused_generated or self.add_missing_images or self.reset_image_cache):
+        if not current_cached_image.is_file() and not (self.allow_unused_generated or self.add_missing_images or self.reset_image_cache):
             # Raise error since the cached image does not exist and will not be added later
 
             # Save images as needed before error
-            if gen_image_filename is not None:
-                plotter.screenshot(gen_image_filename)
+            if self.generated_image_dir is not None:
+                self._save_generated_image(plotter, image_name=image_name)
             if self.failed_image_dir is not None:
                 self._save_failed_test_images("error", plotter, image_name)
 
             remove_plotter_close_callback()
-            msg = f"{image_filename} does not exist in image cache"
+            msg = f"{current_cached_image} does not exist in image cache"
             raise RegressionFileNotFoundError(msg)
 
-        if (self.add_missing_images and not image_filename.is_file()) or (self.reset_image_cache and not self.reset_only_failed):
-            plotter.screenshot(image_filename)
+        if (self.add_missing_images and not current_cached_image.is_file()) or (self.reset_image_cache and not self.reset_only_failed):
+            plotter.screenshot(current_cached_image)
 
-        if gen_image_filename is not None:
-            plotter.screenshot(gen_image_filename)
+        if self.generated_image_dir is not None:
+            self._save_generated_image(plotter, image_name=image_name)
 
-        if not Path(image_filename).is_file() and self.allow_unused_generated:
+        if not Path(current_cached_image).is_file() and self.allow_unused_generated:
             # Test image has been generated, but cached image does not exist
             # The generated image is considered unused, so exit safely before image
             # comparison to avoid a FileNotFoundError
             return
 
-        if self.failed_image_dir is not None and not Path(image_filename).is_file():
-            # Image comparison will fail, so save image before error
-            self._save_failed_test_images("error", plotter, image_name)
-            remove_plotter_close_callback()
-
-        error = pyvista.compare_images(str(image_filename), plotter)
-
-        if error > allowed_error:
+        test_name_no_prefix = test_name.removeprefix("test_")
+        warn_msg, fail_msg = _test_compare_images(
+            test_name=test_name_no_prefix,
+            test_image=plotter,
+            cached_image=str(current_cached_image),
+            allowed_error=allowed_error,
+            allowed_warning=allowed_warning,
+        )
+
+        # Try again and compare with other cached images
+        if fail_msg and len(cached_image_paths) > 1:
+            # Compare build image to other known valid versions
+            msg_start = "This test has multiple cached images. It initially failed (as above)"
+            for path in cached_image_paths:
+                error = pyvista.compare_images(plotter, str(path))
+                if _check_compare_fail(test_name, error, allowed_error=allowed_error) is None:
+                    # Convert failure into a warning
+                    warn_msg = fail_msg + (f"\n{msg_start} but passed when compared to:\n\t{path}")
+                    fail_msg = None
+                    current_cached_image = path
+                    break
+            else:  # Loop completed - test still fails
+                fail_msg += f"\n{msg_start} and failed again for all images in:\n\t{Path(self.cache_dir, test_name_no_prefix)!s}"
+
+        if fail_msg:
             if self.failed_image_dir is not None:
                 self._save_failed_test_images("error", plotter, image_name)
             if self.reset_only_failed:
                 warnings.warn(
-                    f"{test_name} Exceeded image regression error of "
-                    f"{allowed_error} with an image error equal to: {error}"
-                    f"\nThis image will be reset in the cache.",
+                    f"{fail_msg}\nThis image will be reset in the cache.",
                     stacklevel=2,
                 )
-                plotter.screenshot(image_filename)
+                plotter.screenshot(current_cached_image)
             else:
                 remove_plotter_close_callback()
-                msg = f"{test_name} Exceeded image regression error of {allowed_error} with an image error equal to: {error}"
-                raise RegressionError(msg)
-        if error > allowed_warning:
+                raise RegressionError(fail_msg)
+
+        if warn_msg:
+            parent_dir: Literal["errors_as_warning", "warning"] = "errors_as_warning" if image_dirname.is_dir() else "warning"
             if self.failed_image_dir is not None:
-                self._save_failed_test_images("warning", plotter, image_name)
-            warnings.warn(f"{test_name} Exceeded image regression warning of {allowed_warning} with an image error of {error}", stacklevel=2)
+                self._save_failed_test_images(parent_dir, plotter, image_name, cache_image_path=current_cached_image)
+            warnings.warn(warn_msg, stacklevel=2)
 
-    def _save_failed_test_images(self, error_or_warning: Literal["error", "warning"], plotter: Plotter, image_name: str) -> None:
+    def _save_generated_image(self, plotter: pyvista.Plotter, image_name: str, parent_dir: Path | None = None) -> None:
+        parent = cast("Path", self.generated_image_dir) if parent_dir is None else parent_dir
+        generated_image_path = parent / Path(image_name).with_suffix("") / (ENV_INFO + ".png") if self.generate_subdirs else parent / image_name
+        generated_image_path.parent.mkdir(exist_ok=True, parents=True)
+        plotter.screenshot(generated_image_path)
+
+    def _save_failed_test_images(
+        self,
+        error_or_warning: Literal["error", "warning", "errors_as_warning"],
+        plotter: Plotter,
+        image_name: str,
+        cache_image_path: Path | None = None,
+    ) -> None:
         """Save test image from cache and from test to the failed image dir."""
 
         def _make_failed_test_image_dir(
-            errors_or_warnings: Literal["errors", "warnings"], from_cache_or_test: Literal["from_cache", "from_test"]
+            errors_or_warnings: Literal["errors", "warnings", "errors_as_warnings"], from_cache_or_test: Literal["from_cache", "from_test"]
         ) -> Path:
             # Check was done earlier to verify this is not None
             failed_image_dir = cast("str", self.failed_image_dir)
-            _ensure_dir_exists(failed_image_dir, msg_name="failed image dir")
             dest_dir = Path(failed_image_dir, errors_or_warnings, from_cache_or_test)
             dest_dir.mkdir(exist_ok=True, parents=True)
             return dest_dir
 
-        error_dirname = cast("Literal['errors', 'warnings']", error_or_warning + "s")
+        def _save_single_cache_image(path: Path) -> None:
+            rel = Path(path).relative_to(self.cache_dir)
+            from_cache_dir = _make_failed_test_image_dir(error_dirname, "from_cache")
+            dest_path = from_cache_dir / rel
+            dest_path.parent.mkdir(exist_ok=True, parents=True)
+            shutil.copy(path, dest_path)
+
+        error_dirname = cast("Literal['errors', 'warnings', 'errors_as_warnings']", error_or_warning + "s")
 
         from_test_dir = _make_failed_test_image_dir(error_dirname, "from_test")
-        plotter.screenshot(from_test_dir / image_name)
+        self._save_generated_image(plotter, image_name=image_name, parent_dir=from_test_dir)
 
-        cached_image = Path(self.cache_dir, image_name)
+        cached_image = Path(self.cache_dir, image_name) if cache_image_path is None else cache_image_path
         if cached_image.is_file():
-            from_cache_dir = _make_failed_test_image_dir(error_dirname, "from_cache")
-            shutil.copy(cached_image, from_cache_dir / image_name)
+            # Save single cache file
+            _save_single_cache_image(cached_image)
+        elif (image_dir := cached_image.with_suffix("")).is_dir():
+            # Save multiple cached files
+            for path in _get_file_paths(image_dir, ext="png"):
+                _save_single_cache_image(path)
 
 
 def _image_name_from_test_name(test_name: str) -> str:
@@ -354,6 +421,33 @@ def remove_suffix(s: str) -> str:
     return "test_" + remove_suffix(image_name)
 
 
+def _get_file_paths(dir_: Path, ext: str) -> list[Path]:
+    """Get all paths of files with a specific extension inside a directory tree."""
+    return sorted(dir_.rglob(f"*.{ext}"))
+
+
+def _test_compare_images(
+    test_name: str, test_image: str | pyvista.Plotter, cached_image: str | pyvista.Plotter, allowed_error: float, allowed_warning: float
+) -> tuple[str | None, str | None]:
+    # Check if test should fail or warn
+    error = pyvista.compare_images(test_image, cached_image)
+    fail_msg = _check_compare_fail(test_name, error, allowed_error)
+    warn_msg = _check_compare_warn(test_name, error, allowed_warning)
+    return warn_msg, fail_msg
+
+
+def _check_compare_fail(test_name: str, error_: float, allowed_error: float) -> str | None:
+    if error_ > allowed_error:
+        return f"{test_name} Exceeded image regression error of {allowed_error} with an image error equal to: {error_}"
+    return None
+
+
+def _check_compare_warn(test_name: str, error_: float, allowed_warning: float) -> str | None:
+    if error_ > allowed_warning:
+        return f"{test_name} Exceeded image regression warning of {allowed_warning} with an image error of {error_}"
+    return None
+
+
 @pytest.hookimpl
 def pytest_terminal_summary(terminalreporter, exitstatus, config) -> None:  # noqa: ANN001, ARG001
     """Execute after the whole test run completes."""
@@ -471,6 +565,7 @@ def verify_image_cache(
     VerifyImageCache.allow_unused_generated = pytestconfig.getoption("allow_unused_generated")
     VerifyImageCache.add_missing_images = pytestconfig.getoption("add_missing_images")
     VerifyImageCache.reset_only_failed = pytestconfig.getoption("reset_only_failed")
+    VerifyImageCache.generate_subdirs = pytestconfig.getoption("generate_subdirs")
 
     cache_dir = _get_option_from_config_or_ini(pytestconfig, "image_cache_dir", is_dir=True)
     gen_dir = _get_option_from_config_or_ini(pytestconfig, "generated_image_dir", is_dir=True)