diff --git a/README.rst b/README.rst index 494985b..02e5ac3 100644 --- a/README.rst +++ b/README.rst @@ -59,6 +59,15 @@ Alternatively, you can also install via `conda`_ or `mamba`_ from `conda-forge`_ Usage ----- +The plugin has two main use cases: + +#. Evaluate images generated by calling ``pl.show()`` in unit tests +#. Evaluate images generated by the + `Sphinx PyVista Plot Directive `_ + when building documentation. + +Unit tests +---------- Once installed, you only need to use the command `pl.show()` in your test. The plugin will automatically manage the cache generation if it does not exist, and the image comparison itself. Make sure you enable `pv.OFF_SCREEN` when loading @@ -139,7 +148,7 @@ compared to all other cached images for that test. The test is successful if one of the comparisons is successful, though a warning is still issued. Global flags ------------- +============ These are the flags you can use when calling ``pytest`` in the command line: * ``--reset_image_cache`` creates a new image for each test in @@ -189,7 +198,7 @@ These are the flags you can use when calling ``pytest`` in the command line: by the same name below to configure this on a per-test basis. Test specific flags -------------------- +=================== These are attributes of `verify_image_cache`. You can set them as ``True`` if needed in the beginning of your test function. @@ -211,6 +220,111 @@ in the beginning of your test function. ``verify_image_cache`` fixture is used but no images are generated. The value of this flag takes precedence over the global flag by the same name (see above). +Documentation image tests +------------------------- +Unlike the unit tests, which use the ``verify_image_cache`` fixture to evaluate test +images during each test's teardown, the documentation tests do not use a fixture. +Instead, the documentation tests are invoked with the ``--doc_mode`` flag, and requires +specifying: + +#. A target directory which contains all images to be tested. The directory is specified + using the ``--doc_images_dir`` flag. +#. A cache directory containing all reference images to compare with. The directory is + specified using the ``--doc_image_cache_dir`` flag. + +Since all images must be available for testing, the documentation tests are typically +executed `after` building documentation with Sphinx or some other build process. To test +build images against images in a cache directory use: + +.. code-block:: bash + + pytest --doc_mode --doc_images_dir images --doc_image_cache_dir cache + +where ``images`` is the target directory of images to test, and ``cache`` is the cache +directory. + +When executed, the test will first pre-process the build images. The images are: + +#. Collected from the ``images`` directory (including images in nested directories). +#. Resized to a maximum of 400x400 pixels. +#. Renamed so that each file's parent directories are included in the name. +#. Saved as JPEG images with in a temporary directory. The directory is flat with no + sub-directories. + + .. note:: + These temporary images may be saved using the ``--doc_generated_image_dir`` flag. + +Next, the pre-processed images are compared to the cached images. +The tests have three main modes of failure: + +#. An image is in the cache but is missing from the build. +#. An image is in the build but is missing from the cache. +#. The error threshold when comparing two images is exceeded. + +.. note:: + Use the ``--doc_failed_image_dir`` flag to save copies of the images for + failed tests. + +Specifying multiple cache images +================================ +The cache directory is typically flat with no sub-directories. However, +it is possible to specify multiple cache images for a single test by +including a sub-directory with the same name as the image, and including +multiple "valid" cache images in the sub-directory. For example, a +single cached image: + +.. code-block:: bash + + cache/my_image.jpg + +Can be replaced with multiple versions of the image: + +.. code-block:: bash + + cache/my_image/0.jpg + cache/my_image/1.jpg + +.. note:: + + The sub-directory name should match the image name generated from the build. + The image names in sub-directories can be arbitrary, however, e.g. ``0.jpg`` or + ``foo.jpg``, and can even be nested in sub-sub-directories (the names + of sub-sub-directories can also be arbitrary). + +When there are multiple images, the test will initially compare the build image +to the first cached image. If that comparison fails, the build image is then +compared to all other cached images for that test. The test is successful if one +of the comparisons is successful, though a warning is still issued. + +Global flags +============ +These are the flags you can use when calling ``pytest`` in the command line: + +* ``--doc_mode`` is a required flag for testing documentation images. It configures + pytest to only collect tests relevant for the image testing. + +* ``--doc_images_dir ``> sets the target directory of images to be tested. + + .. note:: + + With Sphinx, build images are typically saved to ``doc/_build/html/_images``. + +* ``--doc_image_cache_dir `` sets the doc image cache directory, relative to `pytest root path `. + This will override any configuration, see below. + +* ``--doc_generated_image_dir `` dumps all doc generated test images into the provided + directory, relative to `pytest root path `. + This will override any configuration, see below. + + .. note:: + These are the pre-processed images generated for the tests. They are `not` the images + generated by a documentation build (use ``--doc_images_dir`` for specifying that). + +* ``--doc_failed_image_dir `` dumps copies of cached and generated test images when + there is a warning or error raised. This directory is useful for reviewing test + failures. It is relative to `pytest root path `. + This will override any configuration, see below. + Configuration ------------- If using ``pyproject.toml`` or any other @@ -238,6 +352,16 @@ Similarly, configure the directory that will contain any failed test images: [tool.pytest.ini_options] failed_image_dir = "failed_images" +Configure directories for when ``--doc_mode`` is used: + +.. code-block:: toml + + [tool.pytest.ini_options] + doc_failed_image_dir = "failed_test_images" + doc_generated_image_dir = "generated_test_images" + doc_image_cache_dir = "tests/doc/doc_image_cache" + doc_images_dir = "doc/_build/html/_images" + Note that these directories are relative to `pytest root path `. Contributing diff --git a/pytest_pyvista/doc_mode.py b/pytest_pyvista/doc_mode.py new file mode 100644 index 0000000..948939f --- /dev/null +++ b/pytest_pyvista/doc_mode.py @@ -0,0 +1,297 @@ +"""Test the images generated from building the documentation.""" + +from __future__ import annotations + +from pathlib import Path +import shutil +import tempfile +from typing import ClassVar +from typing import Literal +from typing import NamedTuple +from typing import cast +import warnings + +from PIL import Image +import pytest +import pyvista as pv + +from .pytest_pyvista import _get_file_paths +from .pytest_pyvista import _get_option_from_config_or_ini + +MAX_IMAGE_DIM = 400 # pixels + + +class _DocModeInfo: + doc_images_dir: Path + doc_image_cache_dir: Path + doc_generated_image_dir: Path + doc_failed_image_dir: Path + _tempdirs: ClassVar[list[tempfile.TemporaryDirectory]] = [] + + @classmethod + def init_dirs(cls, config: pytest.Config) -> None: + def require_existing_dir(option: str) -> Path: + """Fetch a required directory option and ensure it's valid.""" + path = _get_option_from_config_or_ini(config, option, is_dir=True) + if path is None: + msg = f"{option!r} must be specified when using --doc_mode" + raise ValueError(msg) + if not path.is_dir(): + msg = f"{option!r} must be a valid directory. Got:\n{path}." + raise ValueError(msg) + return path + + def optional_dir_with_temp(option: str, prefix: str) -> Path: + """Fetch an optional directory option or create a TemporaryDirectory if missing.""" + path = _get_option_from_config_or_ini(config, option, is_dir=True) + if path is None: + tempdir = tempfile.TemporaryDirectory(prefix=prefix) + cls._tempdirs.append(tempdir) + return Path(tempdir.name) + return path + + cls.doc_images_dir = require_existing_dir("doc_images_dir") + cls.doc_image_cache_dir = require_existing_dir("doc_image_cache_dir") + + cls.doc_generated_image_dir = optional_dir_with_temp("doc_generated_image_dir", prefix="pytest_doc_generated_image_dir") + cls.doc_failed_image_dir = optional_dir_with_temp("doc_failed_image_dir", prefix="pytest_doc_failed_image_dir") + + +class _TestCaseTuple(NamedTuple): + test_name: str + docs_image_path: Path + cached_image_path: Path + + +def _flatten_path(path: Path) -> Path: + return Path("_".join(path.parts)) + + +def _preprocess_build_images(build_images_dir: Path, output_dir: Path) -> list[Path]: + """ + Read images from the build dir, resize them, and save as JPG to a flat output dir. + + All PNG and GIF files from the build are included, and are saved as JPG. + + """ + input_png = _get_file_paths(build_images_dir, ext="png") + input_gif = _get_file_paths(build_images_dir, ext="gif") + input_jpg = _get_file_paths(build_images_dir, ext="jpg") + output_paths = [] + output_dir.mkdir(exist_ok=True) + for input_path in input_png + input_gif + input_jpg: + # input image from the docs may come from a nested directory, + # so we flatten the file's relative path + output_file_name = _flatten_path(input_path.relative_to(build_images_dir)) + output_file_name = output_file_name.with_suffix(".jpg") + output_path = output_dir / output_file_name + output_paths.append(output_path) + + # Ensure image size is max 400x400 and save to output + with Image.open(input_path) as im: + im = im.convert("RGB") if im.mode != "RGB" else im # noqa: PLW2901 + if not (im.size[0] <= MAX_IMAGE_DIM and im.size[1] <= MAX_IMAGE_DIM): + im.thumbnail(size=(MAX_IMAGE_DIM, MAX_IMAGE_DIM)) + im.save(output_path, quality="keep") if im.format == "JPEG" else im.save(output_path) + + return output_paths + + +def _generate_test_cases() -> list[_TestCaseTuple]: + """ + Generate a list of image test cases. + + This function: + (1) Generates a list of test images from the docs + (2) Generates a list of cached images + (3) Merges the two lists together and returns separate test cases to + comparing all docs images to all cached images + """ + test_cases_dict: dict = {} + + def add_to_dict(filepath: Path, key: str) -> None: + # Function for stuffing image paths into a dict. + # We use a dict to allow for any entry to be made based on image path alone. + # This way, we can defer checking for any mismatch between the cached and docs + # images to test time. + nonlocal test_cases_dict + test_name = filepath.stem + try: + test_cases_dict[test_name] + except KeyError: + test_cases_dict[test_name] = {} + test_cases_dict[test_name].setdefault(key, filepath) + + # process test images + test_image_paths = _preprocess_build_images(_DocModeInfo.doc_images_dir, _DocModeInfo.doc_generated_image_dir) + [add_to_dict(path, "docs") for path in test_image_paths] # type: ignore[func-returns-value] + + # process cached images + cache_dir = _DocModeInfo.doc_image_cache_dir + cached_image_paths = _get_file_paths(cache_dir, ext="jpg") + for path in cached_image_paths: + # Check if we have a single image or a dir with multiple images + rel = path.relative_to(cache_dir) + parts = rel.parts + if len(parts) > 1: # means it's nested + # Use the first subdir as the test input instead of the image path + first_subdir = parts[0] # one dir down from base + add_to_dict(cache_dir / first_subdir, "cached") + else: + add_to_dict(path, "cached") + + # flatten dict + test_cases_list = [] + for test_name, content in sorted(test_cases_dict.items()): + doc = content.get("docs", None) + cache = content.get("cached", None) + test_case = _TestCaseTuple( + test_name=test_name, + docs_image_path=doc, + cached_image_path=cache, + ) + test_cases_list.append(test_case) + + return test_cases_list + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + """Generate parametrized tests.""" + if "test_case" in metafunc.fixturenames: + # Generate a separate test case for each image being tested + test_cases = _generate_test_cases() + ids = [case.test_name for case in test_cases] + metafunc.parametrize("test_case", test_cases, ids=ids) + + +def _save_failed_test_image(source_path: Path, category: Literal["warnings", "errors", "errors_as_warnings"]) -> None: + """Save test image from cache or build to the failed image dir.""" + _DocModeInfo.doc_failed_image_dir.mkdir(exist_ok=True) + + if source_path.is_relative_to(_DocModeInfo.doc_image_cache_dir): + rel = source_path.relative_to(_DocModeInfo.doc_image_cache_dir) + dest_relative_dir = Path("from_cache") / rel.parent + else: + dest_relative_dir = Path("from_build") + + dest_dir = _DocModeInfo.doc_failed_image_dir / category / dest_relative_dir + dest_dir.mkdir(exist_ok=True, parents=True) + dest_path = dest_dir / source_path.name + shutil.copy(source_path, dest_path) + + +def test_static_images(test_case: _TestCaseTuple) -> None: + """Compare generated image with cached image.""" + _warn_cached_image_path(test_case.cached_image_path) + fail_msg, fail_source = _test_both_images_exist( + filename=test_case.test_name, docs_image_path=test_case.docs_image_path, cached_image_path=test_case.cached_image_path + ) + if fail_msg: + _save_failed_test_image(cast("Path", fail_source), "errors") + pytest.fail(fail_msg) + + cached_image_paths = ( + [test_case.cached_image_path] if test_case.cached_image_path.is_file() else _get_file_paths(test_case.cached_image_path, ext="jpg") + ) + current_cached_image_path = cached_image_paths[0] + + warn_msg, fail_msg = _test_compare_images( + test_name=test_case.test_name, docs_image_path=test_case.docs_image_path, cached_image_path=current_cached_image_path + ) + + # Try again and compare with other cached images + if fail_msg and len(cached_image_paths) > 1: + # Compare build image to other known valid versions + msg_start = "This test has multiple cached images. It initially failed (as above)" + for path in cached_image_paths: + error = pv.compare_images(pv.read(test_case.docs_image_path), pv.read(path)) + if _check_compare_fail(test_case.test_name, error) is None: + # Convert failure into a warning + warn_msg = fail_msg + (f"\n{msg_start} but passed when compared to:\n\t{path}") + fail_msg = None + current_cached_image_path = path + break + else: # Loop completed - test still fails + fail_msg += f"\n{msg_start} and failed again for all images in:\n\t{_DocModeInfo.doc_image_cache_dir / test_case.test_name!s}" + + if fail_msg: + _save_failed_test_image(test_case.docs_image_path, "errors") + # Save all cached images since they all failed + for path in cached_image_paths: + _save_failed_test_image(path, "errors") + pytest.fail(fail_msg) + + if warn_msg: + parent_dir: Literal["errors_as_warnings", "warnings"] = "errors_as_warnings" if test_case.cached_image_path.is_dir() else "warnings" + _save_failed_test_image(test_case.docs_image_path, parent_dir) + _save_failed_test_image(current_cached_image_path, parent_dir) + warnings.warn(warn_msg, stacklevel=2) + + +def _test_both_images_exist(filename: str, docs_image_path: Path, cached_image_path: Path) -> tuple[str | None, Path | None]: + if docs_image_path is None or cached_image_path is None: + if docs_image_path is None: + source_path = cached_image_path + exists = "cache" + missing = "docs build" + exists_path = cached_image_path + missing_path = _DocModeInfo.doc_images_dir + else: + source_path = docs_image_path + exists = "docs build" + missing = "cache" + exists_path = _DocModeInfo.doc_images_dir + missing_path = _DocModeInfo.doc_image_cache_dir + + msg = ( + f"Test setup failed for test image:\n" + f"\t{filename}\n" + f"The image exists in the {exists} directory:\n" + f"\t{exists_path}\n" + f"but is missing from the {missing} directory:\n" + f"\t{missing_path}\n" + ) + return msg, source_path + return None, None + + +def _warn_cached_image_path(cached_image_path: Path) -> None: + """Warn if a subdir is used with only one cached image.""" + if cached_image_path is not None and cached_image_path.is_dir(): + cached_images = _get_file_paths(cached_image_path, ext="jpg") + if len(cached_images) == 1: + cache_dir = _DocModeInfo.doc_image_cache_dir + rel_path = cache_dir.name / cached_images[0].relative_to(cache_dir) + msg = ( + "Cached image sub-directory only contains a single image.\n" + f"Move the cached image {rel_path.as_posix()!r} directly to the cached image dir {cache_dir.name!r}\n" + f"or include more than one image in the sub-directory." + ) + warnings.warn(msg, stacklevel=2) + + +def _test_compare_images(test_name: str, docs_image_path: Path, cached_image_path: Path) -> tuple[str | None, str | None]: + try: + docs_image = cast("pv.ImageData", pv.read(docs_image_path)) + cached_image = cast("pv.ImageData", pv.read(cached_image_path)) + + # Check if test should fail or warn + error = pv.compare_images(docs_image, cached_image) + fail_msg = _check_compare_fail(test_name, error) + warn_msg = _check_compare_warn(test_name, error) + except RuntimeError as e: + warn_msg = None + fail_msg = repr(e) + return warn_msg, fail_msg + + +def _check_compare_fail(filename: str, error_: float, allowed_error: float = 500.0) -> str | None: + if error_ > allowed_error: + return f"{filename} Exceeded image regression error of {allowed_error} with an image error equal to: {error_}" + return None + + +def _check_compare_warn(filename: str, error_: float, allowed_warning: float = 200.0) -> str | None: + if error_ > allowed_warning: + return f"{filename} Exceeded image regression warning of {allowed_warning} with an image error of {error_}" + return None diff --git a/pytest_pyvista/pytest_pyvista.py b/pytest_pyvista/pytest_pyvista.py index 4995145..35d6c5b 100644 --- a/pytest_pyvista/pytest_pyvista.py +++ b/pytest_pyvista/pytest_pyvista.py @@ -3,6 +3,7 @@ from __future__ import annotations import contextlib +import importlib import json import os from pathlib import Path @@ -65,6 +66,8 @@ class RegressionFileNotFoundError(RegressionFileNotFound): def pytest_addoption(parser) -> None: # noqa: ANN001 """Adds new flag options to the pyvista plugin.""" # noqa: D401 + _add_common_pytest_options(parser) + group = parser.getgroup("pyvista") group.addoption( "--reset_image_cache", @@ -77,46 +80,16 @@ def pytest_addoption(parser) -> None: # noqa: ANN001 action="store_true", help="Prevent test failure if a generated test image has no use.", ) - group.addoption( - "--generated_image_dir", - action="store", - help="Path to dump test images from the current run.", - ) - parser.addini( - "generated_image_dir", - default=None, - help="Path to dump test images from the current run.", - ) group.addoption( "--generate_subdirs", action="store_true", help="Save generated images to sub-directories.", ) - group.addoption( - "--failed_image_dir", - action="store", - help="Path to dump images from failed tests from the current run.", - ) - parser.addini( - "failed_image_dir", - default=None, - help="Path to dump images from failed tests from the current run.", - ) group.addoption( "--add_missing_images", action="store_true", help="Adds images to cache if missing.", ) - group.addoption( - "--image_cache_dir", - action="store", - help="Path to the image cache folder.", - ) - parser.addini( - "image_cache_dir", - default="image_cache_dir", - help="Path to the image cache folder.", - ) group.addoption( "--reset_only_failed", action="store_true", @@ -133,6 +106,59 @@ def pytest_addoption(parser) -> None: # noqa: ANN001 help="Prevent test failure if the `verify_image_cache` fixture is used but no images are generated.", ) + # Doc-specific test options + group.addoption( + "--doc_mode", + action="store_true", + help="Enable documentation image testing.", + ) + group.addoption( + "--doc_images_dir", + action="store", + help="Path to the documentation images.", + ) + parser.addini( + "doc_images_dir", + default=None, + help="Path to the documentation images.", + ) + _add_common_pytest_options(parser, doc=True) + + +def _add_common_pytest_options(parser, *, doc: bool = False) -> None: # noqa: ANN001 + prefix = "doc_" if doc else "" + group = parser.getgroup("pyvista") + group.addoption( + f"--{prefix}image_cache_dir", + action="store", + help="Path to the image cache folder.", + ) + parser.addini( + f"{prefix}image_cache_dir", + default=None if doc else "image_cache_dir", + help="Path to the image cache folder.", + ) + group.addoption( + f"--{prefix}generated_image_dir", + action="store", + help="Path to dump test images from the current run.", + ) + parser.addini( + f"{prefix}generated_image_dir", + default=None, + help="Path to dump test images from the current run.", + ) + group.addoption( + f"--{prefix}failed_image_dir", + action="store", + help="Path to dump images from failed tests from the current run.", + ) + parser.addini( + f"{prefix}failed_image_dir", + default=None, + help="Path to dump images from failed tests from the current run.", + ) + class VerifyImageCache: """ @@ -456,7 +482,8 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config) -> None: # no return if config.getoption("disallow_unused_cache"): - cache_path = Path(_get_option_from_config_or_ini(config, "image_cache_dir")) + value = _get_option_from_config_or_ini(config, "image_cache_dir") + cache_path = Path(cast("Path", value)) cached_image_names = {f.name for f in cache_path.glob("*.png")} image_names_dir = getattr(config, "image_names_dir", None) @@ -501,7 +528,7 @@ def _ensure_dir_exists(dirpath: str | Path, msg_name: str) -> None: Path(dirpath).mkdir(exist_ok=True, parents=True) -def _get_option_from_config_or_ini(pytestconfig: pytest.Config, option: str, *, is_dir: bool = False): # noqa: ANN202 +def _get_option_from_config_or_ini(pytestconfig: pytest.Config, option: str, *, is_dir: bool = False) -> Path | None: value = pytestconfig.getoption(option) if value is None: value = pytestconfig.getini(option) @@ -541,6 +568,11 @@ def __call__(self, plotter: Plotter) -> None: @pytest.hookimpl def pytest_configure(config: pytest.Config) -> None: """Configure pytest session.""" + if config.getoption("doc_mode"): + from pytest_pyvista.doc_mode import _DocModeInfo # noqa: PLC0415 + + _DocModeInfo.init_dirs(config) + # create a image names directory for individual or multiple workers to write to if config.getoption("disallow_unused_cache"): config.image_names_dir = Path(config.cache.makedir("pyvista")) @@ -567,7 +599,7 @@ def verify_image_cache( VerifyImageCache.reset_only_failed = pytestconfig.getoption("reset_only_failed") VerifyImageCache.generate_subdirs = pytestconfig.getoption("generate_subdirs") - cache_dir = _get_option_from_config_or_ini(pytestconfig, "image_cache_dir", is_dir=True) + cache_dir = cast("Path", _get_option_from_config_or_ini(pytestconfig, "image_cache_dir", is_dir=True)) gen_dir = _get_option_from_config_or_ini(pytestconfig, "generated_image_dir", is_dir=True) failed_dir = _get_option_from_config_or_ini(pytestconfig, "failed_image_dir", is_dir=True) @@ -650,3 +682,36 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: # n # Fixed: Write JSON instead of plain text visited_file.write_text(json.dumps(list(VISITED_CACHED_IMAGE_NAMES))) skipped_file.write_text(json.dumps(list(SKIPPED_CACHED_IMAGE_NAMES))) + + +def pytest_unconfigure(config: pytest.Config) -> None: + """Remove temporary files.""" + if config.getoption("doc_mode"): + from pytest_pyvista.doc_mode import _DocModeInfo # noqa: PLC0415 + + for tempdir in _DocModeInfo._tempdirs: # noqa: SLF001 + tempdir.cleanup() + _DocModeInfo._tempdirs = [] # noqa: SLF001 + + +def pytest_ignore_collect(collection_path: Path, config: pytest.Config) -> bool | None: # noqa: ARG001 + """Block regular file collection entirely when using --doc_mode.""" + if config.getoption("doc_mode"): + return True + return None + + +def pytest_collection_modifyitems(session: pytest.Session, config: pytest.Config, items: list[pytest.Item]) -> None: + """Collect tests from doc images when --doc_mode is enabled.""" + if config.getoption("doc_mode"): + items.clear() # Clear previously collected items + + # Import the doc images module + module_name = "pytest_pyvista.doc_mode" + doc_module = importlib.import_module(module_name) + module_file = Path(cast("Path", doc_module.__file__)) + + # Collect test items from the module + module_collector = pytest.Module.from_parent(parent=session, path=module_file) + collected_items = list(module_collector.collect()) + items.extend(collected_items) diff --git a/tests/test_doc_mode.py b/tests/test_doc_mode.py new file mode 100644 index 0000000..6e230e8 --- /dev/null +++ b/tests/test_doc_mode.py @@ -0,0 +1,284 @@ +"""Test the --doc_mode option.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from PIL import Image +import pytest +import pyvista as pv + +from pytest_pyvista.doc_mode import _preprocess_build_images +from tests.test_pyvista import file_has_changed +from tests.test_pyvista import make_cached_images +from tests.test_pyvista import make_multiple_cached_images + + +@pytest.mark.parametrize("generated_image_dir", [True, False]) +def test_doc_mode(pytester: pytest.Pytester, generated_image_dir) -> None: + """Test regular usage of the --doc_mode.""" + cache = "cache" + images = "images" + make_cached_images(pytester.path, cache) + make_cached_images(pytester.path, images) + _preprocess_build_images(pytester.path / cache, pytester.path / cache) + + args = ["--doc_mode", "--doc_images_dir", images, "--doc_image_cache_dir", cache] + generated = "generated" + if generated_image_dir: + args.extend(["--doc_generated_image_dir", generated]) + result = pytester.runpytest(*args) + assert result.ret == pytest.ExitCode.OK + + if generated_image_dir: + assert Path(generated).is_dir() + assert os.listdir(generated) == ["imcache.jpg"] # noqa: PTH208 + + +def test_cli_errors(pytester: pytest.Pytester) -> None: + """Test errors generated when using CLI.""" + result = pytester.runpytest("--doc_mode") + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + result.stderr.fnmatch_lines(["*ValueError: 'doc_images_dir' must be specified when using --doc_mode"]) + + images_path = pytester.path / "images" + result = pytester.runpytest("--doc_mode", "--doc_images_dir", str(images_path)) + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + result.stderr.fnmatch_lines(["*ValueError: 'doc_images_dir' must be a valid directory. Got:", "*/images."]) + + images_path.mkdir() + result = pytester.runpytest("--doc_mode", "--doc_images_dir", str(images_path)) + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + result.stderr.fnmatch_lines(["*ValueError: 'doc_image_cache_dir' must be specified when using --doc_mode"]) + + cache_path = pytester.path / "cache" + result = pytester.runpytest("--doc_mode", "--doc_images_dir", str(images_path), "--doc_image_cache_dir", str(cache_path)) + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + result.stderr.fnmatch_lines(["*ValueError: 'doc_image_cache_dir' must be a valid directory. Got:", "*/cache."]) + + +@pytest.mark.parametrize("missing", ["build", "cache"]) +def test_both_images_exist(pytester: pytest.Pytester, missing: str) -> None: + """Test when either the cache or build image is missing for the test.""" + images_path = pytester.path / "images" + cache_path = pytester.path / "cache" + if missing == "build": + make_cached_images(cache_path.parent, cache_path.name) + _preprocess_build_images(cache_path, cache_path) + expected_lines = [ + "*The image exists in the cache directory:", + f"*{cache_path.name}/imcache.jpg", + "*but is missing from the docs build directory:", + f"*{images_path.name}", + ] + else: + make_cached_images(images_path.parent, images_path.name) + expected_lines = [ + "*The image exists in the docs build directory:", + f"*{images_path.name}", + "*but is missing from the cache directory:", + f"*{cache_path.name}", + ] + + images_path.mkdir(exist_ok=True) + cache_path.mkdir(exist_ok=True) + result = pytester.runpytest("--doc_mode", "--doc_images_dir", images_path, "--doc_image_cache_dir", cache_path) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines(["*Failed: Test setup failed for test image:*", *expected_lines]) + + +def test_compare_images_with_different_sizes(pytester: pytest.Pytester) -> None: + """Test error is raised when there is a mismatch in image size.""" + cache = "cache" + images = "images" + make_cached_images(pytester.path, cache) + make_cached_images(pytester.path, images) + + file = pytester.path / cache / "imcache.png" + with Image.open(file) as im: + im = im.convert("RGB") if im.mode != "RGB" else im # noqa: PLW2901 + im.save(file.with_suffix(".jpg")) + + result = pytester.runpytest("--doc_mode", "--doc_images_dir", images, "--doc_image_cache_dir", cache) + assert result.ret == pytest.ExitCode.TESTS_FAILED + result.stdout.fnmatch_lines(["*Failed: RuntimeError('Input images are not the same size.')"]) + + +def test_compare_images_error(pytester: pytest.Pytester) -> None: + """Test regression error is raised.""" + cache = "cache" + images = "images" + make_cached_images(pytester.path, cache, color="red") + make_cached_images(pytester.path, images, color="blue") + _preprocess_build_images(pytester.path / cache, pytester.path / cache) + + result = pytester.runpytest("--doc_mode", "--doc_images_dir", images, "--doc_image_cache_dir", cache) + assert result.ret == pytest.ExitCode.TESTS_FAILED + + result.stdout.re_match_lines([r".*Failed: imcache Exceeded image regression error of 500\.0 with an image error equal to: [0-9]+\.[0-9]+"]) + + +@pytest.mark.parametrize("failed_image_dir", [True, False]) +def test_compare_images_warning(pytester: pytest.Pytester, *, failed_image_dir: bool) -> None: + """Test regression warning is issued.""" + cache = "cache" + images = "images" + name = "im.png" + make_cached_images(pytester.path, cache, name=name, color=[255, 0, 0]) + make_cached_images(pytester.path, images, name=name, color=[240, 0, 0]) + _preprocess_build_images(pytester.path / cache, pytester.path / cache) + + args = ["--doc_mode", "--doc_images_dir", images, "--doc_image_cache_dir", cache] + failed = "failed" + if failed_image_dir: + args.extend(["--doc_failed_image_dir", failed]) + result = pytester.runpytest(*args) + assert result.ret == pytest.ExitCode.OK + + # Check images saved to the failed image dir + assert Path(failed).is_dir() == failed_image_dir + assert Path(failed, "warnings").is_dir() == failed_image_dir + if failed_image_dir: + name = str(Path(name).with_suffix(".jpg")) + original = Path(cache, name) + from_cache = Path(failed, "warnings", "from_cache", name) + assert from_cache.is_file() + assert not file_has_changed(str(from_cache), str(original)) + + from_build = Path(failed, "warnings", "from_build", name) + assert from_build.is_file() + assert file_has_changed(str(from_build), str(from_cache)) + + result.stdout.re_match_lines( + [rf".*UserWarning: {Path(name).stem} Exceeded image regression warning of 200\.0 with an image error of [0-9]+\.[0-9]+"] + ) + + +ALMOST_BLUE = [0, 0, 254] +ALMOST_RED = [254, 0, 0] + + +@pytest.mark.parametrize("failed_image_dir", [True, False]) +@pytest.mark.parametrize("nested_subdir", [True, False]) +@pytest.mark.parametrize( + ("build_color", "return_code"), [(ALMOST_RED, pytest.ExitCode.OK), (ALMOST_BLUE, pytest.ExitCode.OK), ("green", pytest.ExitCode.TESTS_FAILED)] +) +def test_multiple_cache_images(pytester: pytest.Pytester, build_color, return_code, nested_subdir, failed_image_dir) -> None: + """Test regression warning is issued.""" + cache = "cache" + images = "images" + name = "imcache.png" + subdir = Path(name).stem + cache_parent = pytester.path / cache + cache_parent = cache_parent / subdir if nested_subdir else cache_parent + red_filename = make_cached_images(cache_parent, subdir, name="im1.png", color="red") + blue_filename = make_cached_images(cache_parent, subdir, name="im2.png", color="blue") + build_filename = make_cached_images(pytester.path, images, name=name, color=build_color) + _preprocess_build_images(cache_parent / subdir, cache_parent / subdir) + + args = ["--doc_mode", "--doc_images_dir", images, "--doc_image_cache_dir", cache] + failed = "failed" + if failed_image_dir: + args.extend(["--doc_failed_image_dir", failed]) + result = pytester.runpytest(*args) + assert result.ret == return_code + + partial_match = r"imcache Exceeded image regression error of 500\.0 with an image error equal to: [0-9]+\.[0-9]+" + rel_subdirs = Path(subdir) / subdir if nested_subdir else subdir + + if build_color == ALMOST_RED: + # Comparison with first image succeeds without issue + result.stdout.no_re_match_line(rf".*UserWarning: {partial_match}") + + # Test no images are saved + assert not Path(failed).is_dir() + + elif build_color == ALMOST_BLUE: + # Comparison with first image fails + # Expect error was converted to a warning + result.stdout.re_match_lines( + [ + rf".*UserWarning: {partial_match}", + r".*This test has multiple cached images. It initially failed \(as above\) but passed when compared to:", + ".*im2.jpg", + ] + ) + # Test failed images are saved + cached_original = blue_filename.with_suffix(".jpg") + from_cache = Path(failed) / "errors_as_warnings" / "from_cache" / rel_subdirs / cached_original.name + assert from_cache.is_file() == failed_image_dir + if failed_image_dir: + assert not file_has_changed(str(from_cache), str(cached_original)) + + from_build = Path(failed, "errors_as_warnings", "from_build", build_filename.with_suffix(".jpg").name) + assert from_build.is_file() == failed_image_dir + if failed_image_dir: + assert file_has_changed(str(from_build), str(from_cache)) + + else: # 'green' + # Comparison with all cached images fails + result.stdout.re_match_lines( + [ + rf".*Failed: {partial_match}", + r".*This test has multiple cached images. It initially failed \(as above\) and failed again for all images in:", + ".*cache/imcache", + ] + ) + + # Test failed images are saved + # Expect both red and blue cached images saved + for filename in [blue_filename, red_filename]: + cached_original = filename.with_suffix(".jpg") + from_cache = Path(failed) / "errors" / "from_cache" / rel_subdirs / cached_original.name + assert from_cache.is_file() == failed_image_dir + if failed_image_dir: + assert not file_has_changed(str(from_cache), str(cached_original)) + + from_build = Path(failed, "errors", "from_build", build_filename.with_suffix(".jpg").name) + assert from_build.is_file() == failed_image_dir + if failed_image_dir: + assert file_has_changed(str(from_build), str(from_cache)) + + +def test_single_cache_image_in_subdir(pytester: pytest.Pytester) -> None: + """Test that a warning is emitting for a cache subdir with only one image.""" + cache = "cache" + images = "images" + subdir = "imcache" + make_cached_images(pytester.path / cache, subdir) + make_cached_images(pytester.path, images) + _preprocess_build_images(pytester.path / cache / subdir, pytester.path / cache / subdir) + + result = pytester.runpytest("--doc_mode", "--doc_images_dir", images, "--doc_image_cache_dir", cache) + assert result.ret == pytest.ExitCode.OK + match = [ + ".*UserWarning: Cached image sub-directory only contains a single image.", + ".*Move the cached image 'cache/imcache/imcache.jpg' directly to the cached image dir 'cache'", + ".*or include more than one image in the sub-directory.", + ] + result.stdout.re_match_lines(match) + + +def test_multiple_cache_images_parallel(pytester: pytest.Pytester) -> None: + """Ensure that doc_mode works with multiple workers.""" + cache = "cache" + images = "images" + + n_images = 50 + make_multiple_cached_images(pytester.path, cache, n_images=n_images) + image_filenames = make_multiple_cached_images(pytester.path, images, n_images=n_images) + + _preprocess_build_images(pytester.path / cache, pytester.path / cache) + + result = pytester.runpytest("--doc_mode", "--doc_images_dir", images, "--doc_image_cache_dir", cache, "-n2") + assert result.ret == pytest.ExitCode.OK + + # replace a single image with a different image + img_idx = 34 + pv.Cube().plot(screenshot=image_filenames[img_idx]) + + result = pytester.runpytest("--doc_mode", "--doc_images_dir", images, "--doc_image_cache_dir", cache, "-n2") + assert result.ret == pytest.ExitCode.TESTS_FAILED + + assert f"imcache{img_idx} Exceeded image regression error" in str(result.stdout) diff --git a/tests/test_pyvista.py b/tests/test_pyvista.py index b62f455..4e54444 100644 --- a/tests/test_pyvista.py +++ b/tests/test_pyvista.py @@ -6,9 +6,11 @@ import filecmp from pathlib import Path import platform +import shutil import sys from unittest import mock +import matplotlib.pyplot as plt import pytest import pyvista as pv @@ -46,6 +48,35 @@ def make_cached_images(test_path, path="image_cache_dir", name="imcache.png", co return filename +def make_multiple_cached_images(test_path, path="image_cache_dir", n_images: int = 10, name: str = "imcache{index}.png") -> list[Path]: + """Make image cache in `test_path/path` consisting of several images.""" + colors = list(plt.rcParams["axes.prop_cycle"].by_key()["color"]) + + d = Path(test_path, path) + d.mkdir(exist_ok=True, parents=True) + + color_to_file = {} + mesh = pv.Sphere() + + filenames = [] + for ii in range(n_images): + color = colors[ii % len(colors)] + filename = d / name.format(index=ii) + + if color in color_to_file: + # don't regenerate images when that color already exists + shutil.copy(color_to_file[color], filename) + else: + plotter = pv.Plotter(off_screen=True) + plotter.add_mesh(mesh, color=color) + plotter.screenshot(filename) + color_to_file[color] = filename + + filenames.append(filename) + + return filenames + + def get_path_inode(path: str | Path) -> int: """Return the inode for the given path.""" return Path(path).stat().st_ino