Consolidate image comparison code between unit tests and doc mode

README.rst

@@ -66,6 +66,43 @@ The plugin has two main use cases:
    `Sphinx PyVista Plot Directive <https://docs.pyvista.org/extras/plot_directive.html>`_
    when building documentation.
 
+Specifying multiple cache images
+================================
+The cache directory is typically flat with no sub-directories. However,
+it is possible to specify multiple cache images for a single test by
+including a sub-directory with the same name as the test, and including
+multiple "valid" cache images in the sub-directory. For example, a
+single cached image:
+
+.. code-block:: bash
+
+    cache/my_image.jpg
+
+Can be replaced with multiple versions of the image:
+
+.. code-block:: bash
+
+    cache/my_image/0.jpg
+    cache/my_image/1.jpg
+
+.. note::
+
+   - The sub-directory name should match the name of the test.
+   - The image names in sub-directories can be arbitrary, e.g. ``0.jpg`` or
+     ``foo.jpg``.
+   - Nested sub-directories are also supported, and their names can also be arbitrary.
+   - Use the ``--generate_subdirs`` flag to automatically generate test images in a
+     sub-directory format.
+
+When there are multiple images, the test will initially compare the test image
+to the first cached image. If that comparison fails, the test image is then
+compared to all other cached images for that test. The test is successful if one
+of the comparisons is successful, though a warning is still issued if it initially
+failed.
+
+Both use cases (i.e. unit tests and documentation tests) support specifying multiple
+cache images.
+
 Unit tests
 ----------
 Once installed, you only need to use the command `pl.show()` in your test. The
@@ -114,39 +151,6 @@ If you need to use any flag inside the tests, you can modify the
         pl.show()
 
 
-Specifying multiple cache images
-================================
-The cache directory is typically flat with no sub-directories. However,
-it is possible to specify multiple cache images for a single test by
-including a sub-directory with the same name as the test, and including
-multiple "valid" cache images in the sub-directory. For example, a
-single cached image:
-
-.. code-block:: bash
-
-    cache/my_image.jpg
-
-Can be replaced with multiple versions of the image:
-
-.. code-block:: bash
-
-    cache/my_image/0.jpg
-    cache/my_image/1.jpg
-
-.. note::
-
-   - The sub-directory name should match the name of the test.
-   - The image names in sub-directories can be arbitrary, e.g. ``0.jpg`` or
-     ``foo.jpg``.
-   - Nested sub-directories are also supported, and their names can also be arbitrary.
-   - Use the ``--generate_subdirs`` flag to automatically generate test images in a
-     sub-directory format.
-
-When there are multiple images, the test will initially compare the build image
-to the first cached image. If that comparison fails, the build image is then
-compared to all other cached images for that test. The test is successful if one
-of the comparisons is successful, though a warning is still issued.
-
 Global flags
 ============
 These are the flags you can use when calling ``pytest`` in the command line:
@@ -295,37 +299,6 @@ The tests have three main modes of failure:
    Use the ``--doc_failed_image_dir`` flag to save copies of the images for
    failed tests.
 
-Specifying multiple cache images
-================================
-The cache directory is typically flat with no sub-directories. However,
-it is possible to specify multiple cache images for a single test by
-including a sub-directory with the same name as the image, and including
-multiple "valid" cache images in the sub-directory. For example, a
-single cached image:
-
-.. code-block:: bash
-
-    cache/my_image.jpg
-
-Can be replaced with multiple versions of the image:
-
-.. code-block:: bash
-
-    cache/my_image/0.jpg
-    cache/my_image/1.jpg
-
-.. note::
-
-   The sub-directory name should match the image name generated from the build.
-   The image names in sub-directories can be arbitrary, however, e.g. ``0.jpg`` or
-   ``foo.jpg``, and can even be nested in sub-sub-directories (the names
-   of sub-sub-directories can also be arbitrary).
-
-When there are multiple images, the test will initially compare the build image
-to the first cached image. If that comparison fails, the build image is then
-compared to all other cached images for that test. The test is successful if one
-of the comparisons is successful, though a warning is still issued.
-
 Global flags
 ============
 These are the flags you can use when calling ``pytest`` in the command line:

pytest_pyvista/doc_mode.py

@@ -15,9 +15,13 @@
 import pytest
 import pyvista as pv
 
+from .pytest_pyvista import DEFAULT_ERROR_THRESHOLD
+from .pytest_pyvista import DEFAULT_WARNING_THRESHOLD
+from .pytest_pyvista import _check_compare_fail
 from .pytest_pyvista import _get_file_paths
 from .pytest_pyvista import _get_option_from_config_or_ini
 from .pytest_pyvista import _ImageFormats
+from .pytest_pyvista import _test_compare_images
 
 MAX_IMAGE_DIM = 400  # pixels
 
@@ -203,16 +207,20 @@ def test_static_images(test_case: _TestCaseTuple) -> None:
     current_cached_image_path = cached_image_paths[0]
 
     warn_msg, fail_msg = _test_compare_images(
-        test_name=test_case.test_name, docs_image_path=test_case.docs_image_path, cached_image_path=current_cached_image_path
+        test_name=test_case.test_name,
+        test_image=test_case.docs_image_path,
+        cached_image=current_cached_image_path,
+        allowed_error=DEFAULT_ERROR_THRESHOLD,
+        allowed_warning=DEFAULT_WARNING_THRESHOLD,
     )
 
     # Try again and compare with other cached images
     if fail_msg and len(cached_image_paths) > 1:
         # Compare build image to other known valid versions
         msg_start = "This test has multiple cached images. It initially failed (as above)"
-        for path in cached_image_paths:
+        for path in cached_image_paths[1:]:
             error = pv.compare_images(pv.read(test_case.docs_image_path), pv.read(path))
-            if _check_compare_fail(test_case.test_name, error) is None:
+            if _check_compare_fail(test_case.test_name, error, allowed_error=DEFAULT_ERROR_THRESHOLD) is None:
                 # Convert failure into a warning
                 warn_msg = fail_msg + (f"\n{msg_start} but passed when compared to:\n\t{path}")
                 fail_msg = None
@@ -275,30 +283,3 @@ def _warn_cached_image_path(cached_image_path: Path) -> None:
                 f"or include more than one image in the sub-directory."
             )
             warnings.warn(msg, stacklevel=2)
-
-
-def _test_compare_images(test_name: str, docs_image_path: Path, cached_image_path: Path) -> tuple[str | None, str | None]:
-    try:
-        docs_image = cast("pv.ImageData", pv.read(docs_image_path))
-        cached_image = cast("pv.ImageData", pv.read(cached_image_path))
-
-        # Check if test should fail or warn
-        error = pv.compare_images(docs_image, cached_image)
-        fail_msg = _check_compare_fail(test_name, error)
-        warn_msg = _check_compare_warn(test_name, error)
-    except RuntimeError as e:
-        warn_msg = None
-        fail_msg = repr(e)
-    return warn_msg, fail_msg
-
-
-def _check_compare_fail(filename: str, error_: float, allowed_error: float = 500.0) -> str | None:
-    if error_ > allowed_error:
-        return f"{filename} Exceeded image regression error of {allowed_error} with an image error equal to: {error_}"
-    return None
-
-
-def _check_compare_warn(filename: str, error_: float, allowed_warning: float = 200.0) -> str | None:
-    if error_ > allowed_warning:
-        return f"{filename} Exceeded image regression warning of {allowed_warning} with an image error of {error_}"
-    return None

pytest_pyvista/pytest_pyvista.py

@@ -34,6 +34,9 @@
 VISITED_CACHED_IMAGE_NAMES: set[str] = set()
 SKIPPED_CACHED_IMAGE_NAMES: set[str] = set()
 
+DEFAULT_ERROR_THRESHOLD: float = 500.0
+DEFAULT_WARNING_THRESHOLD: float = 200.0
+
 _ImageFormats = Literal["png", "jpg"]
 
 
@@ -323,8 +326,8 @@ def __init__(  # noqa: PLR0913
         test_name: str,
         cache_dir: Path,
         *,
-        error_value: float = 500.0,
-        warning_value: float = 200.0,
+        error_value: float = DEFAULT_ERROR_THRESHOLD,
+        warning_value: float = DEFAULT_WARNING_THRESHOLD,
         var_error_value: float = 1000.0,
         var_warning_value: float = 1000.0,
         generated_image_dir: Path | None = None,
@@ -440,7 +443,7 @@ def remove_plotter_close_callback() -> None:
         warn_msg, fail_msg = _test_compare_images(
             test_name=test_name_no_prefix,
             test_image=plotter,
-            cached_image=str(current_cached_image),
+            cached_image=current_cached_image,
             allowed_error=allowed_error,
             allowed_warning=allowed_warning,
         )
@@ -449,7 +452,7 @@ def remove_plotter_close_callback() -> None:
         if fail_msg and len(cached_image_paths) > 1:
             # Compare test image to other known valid versions
             msg_start = "This test has multiple cached images. It initially failed (as above)"
-            for path in cached_image_paths:
+            for path in cached_image_paths[1:]:
                 error = _compare_images(plotter, str(path))
                 if _check_compare_fail(test_name, error, allowed_error=allowed_error) is None:
                     # Convert failure into a warning
@@ -552,23 +555,30 @@ def _get_file_paths(dir_: Path, ext: str) -> list[Path]:
     return sorted(dir_.rglob(f"*.{ext}"))
 
 
-def _compare_images(test_image: str | pyvista.Plotter, cached_image: str) -> float:
+def _compare_images(test_image: Path | str | pyvista.Plotter, cached_image: Path | str) -> float:
+    def _path_as_string(image: Path | str | pyvista.Plotter) -> str | pyvista.Plotter:
+        return str(image) if isinstance(image, Path) else image
+
     if isinstance(test_image, pyvista.Plotter) and (cached_suffix := Path(cached_image).suffix) == ".jpg":
         # Need to save image to file to apply jpg compression
         pl = cast("pyvista.Plotter", test_image)
         with tempfile.NamedTemporaryFile(suffix=cached_suffix) as tmp:
             pl.screenshot(tmp.name)
-            return pyvista.compare_images(tmp.name, cached_image)
-    return pyvista.compare_images(test_image, cached_image)
+            return pyvista.compare_images(tmp.name, _path_as_string(cached_image))
+    return pyvista.compare_images(_path_as_string(test_image), _path_as_string(cached_image))
 
 
 def _test_compare_images(
-    test_name: str, test_image: str | pyvista.Plotter, cached_image: str, allowed_error: float, allowed_warning: float
+    test_name: str, test_image: Path | str | pyvista.Plotter, cached_image: Path | str, allowed_error: float, allowed_warning: float
 ) -> tuple[str | None, str | None]:
-    # Check if test should fail or warn
-    error = _compare_images(test_image, cached_image)
-    fail_msg = _check_compare_fail(test_name, error, allowed_error)
-    warn_msg = _check_compare_warn(test_name, error, allowed_warning)
+    try:
+        # Check if test should fail or warn
+        error = _compare_images(test_image, cached_image)
+        fail_msg = _check_compare_fail(test_name, error, allowed_error)
+        warn_msg = _check_compare_warn(test_name, error, allowed_warning)
+    except RuntimeError as e:
+        warn_msg = None
+        fail_msg = repr(e)
     return warn_msg, fail_msg