From 493f39038424a877f0a04d088c171a546c41ac25 Mon Sep 17 00:00:00 2001 From: John Lyu Date: Thu, 4 Dec 2025 03:02:22 +0000 Subject: [PATCH 1/4] feat: Add filesystem-only caching with global refresh support - Implement CacheManager class for code execution result caching - Support hash-based caching (automatic invalidation on code changes) - Support custom cache IDs for cross-build persistence - Add MARKDOWN_EXEC_CACHE_REFRESH environment variable for global refresh - Add refresh option to force cache updates per code block - Store cache files in .markdown-exec-cache/ directory - Add comprehensive test suite with 14 cache-specific tests - Add caching documentation with usage examples - Update README with caching quickstart - Isolate test cache directories to prevent pollution This improves documentation build performance by caching expensive operations like plot generation, API calls, and computations. --- .gitignore | 1 + README.md | 35 ++ docs/usage/caching.md | 245 ++++++++++ mkdocs.yml | 1 + src/markdown_exec/__init__.py | 3 + src/markdown_exec/_internal/cache.py | 191 ++++++++ .../_internal/formatters/base.py | 76 +++- src/markdown_exec/_internal/main.py | 14 + tests/conftest.py | 30 ++ tests/test_cache.py | 423 ++++++++++++++++++ 10 files changed, 1005 insertions(+), 14 deletions(-) create mode 100644 docs/usage/caching.md create mode 100644 src/markdown_exec/_internal/cache.py create mode 100644 tests/test_cache.py diff --git a/.gitignore b/.gitignore index 9fea047..c8d1a9b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ uv.lock .mypy_cache/ .ruff_cache/ __pycache__/ +.markdown-exec-cache/ diff --git a/README.md b/README.md index 40d9195..7027bd6 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,41 @@ grep extra_css README.md && exit 2 ``` ```` +### Caching + +Speed up your builds by caching execution results: + +````md +```python exec="yes" cache="yes" +# Expensive computation +import time +time.sleep(5) +print("Done!") +``` +```` + +Use custom cache IDs for persistence across builds: + +````md +```python exec="yes" cache="my-plot" +# Generate plot - will be cached +import matplotlib.pyplot as plt +# ... +``` +```` + +Force cache refresh with `refresh="yes"`: + +````md +```python exec="yes" cache="my-plot" refresh="yes" +# This will always re-execute +``` +```` + +See [caching documentation](https://pawamoy.github.io/markdown-exec/usage/caching/) for more details. + +--- + See [usage](https://pawamoy.github.io/markdown-exec/usage/) for more details, and the [gallery](https://pawamoy.github.io/markdown-exec/gallery/) for more examples! diff --git a/docs/usage/caching.md b/docs/usage/caching.md new file mode 100644 index 0000000..d9f36e3 --- /dev/null +++ b/docs/usage/caching.md @@ -0,0 +1,245 @@ +# Caching + +Markdown Exec supports filesystem-based caching of code execution results to speed up documentation builds and development workflows. + +## Overview + +When generating images, charts, or running expensive computations in your documentation, re-executing the same code on every build can significantly slow down the rendering process. The caching feature allows you to: + +- **Speed up builds**: Reuse previously computed results instead of re-executing code +- **Persist across builds**: All cache is stored on the filesystem for cross-build persistence +- **Global cache refresh**: Force refresh of all cached results with a single environment variable + +## Cache Storage + +All cached results are stored in `.markdown-exec-cache/` in your project root directory: + +``` +your-project/ +├── docs/ +├── mkdocs.yml +└── .markdown-exec-cache/ + ├── my-plot.cache # Custom ID cache + └── abc123def456.cache # Hash-based cache files +``` + +Add this directory to your `.gitignore`: + +```gitignore +.markdown-exec-cache/ +``` + +## Usage + +### Hash-Based Caching + +Enable caching by adding `cache="yes"` to your code block. A hash is computed from the code content and execution options: + +````md exec="1" source="tabbed-left" tabs="Markdown|Rendered" +```python exec="yes" cache="yes" +import time +print(f"Executed at: {time.time()}") +``` +```` + +The cache is automatically invalidated when the code or execution options change. + +### Custom Cache IDs + +For more control, use a custom cache ID (string value). This is useful for expensive operations where you want explicit control over cache invalidation: + +````md exec="1" source="tabbed-left" tabs="Markdown|Rendered" +```python exec="yes" cache="my-plot" +import matplotlib.pyplot as plt +# Expensive plot generation... +print("Generated plot") +``` +```` + +The cache file will be stored as `.markdown-exec-cache/my-plot.cache`. + +### Cache Invalidation + +To force re-execution and update the cache for a specific code block, use `refresh="yes"`: + +```markdown +```python exec="yes" cache="my-plot" refresh="yes" +# This will always re-execute and update the cache +print("Fresh execution!") +``` +``` + +!!! note "refresh vs removing cache" + **`refresh="yes"`** forces re-execution but **keeps the cache enabled** - it updates the cached result for future builds. + + **Removing `cache` option** completely disables caching - the code executes every time with no caching at all. + + Use `refresh="yes"` when you want to update stale cache but keep caching benefits for subsequent builds. + +### Global Cache Refresh + +To refresh **all** cached results at once, set the `MARKDOWN_EXEC_CACHE_REFRESH` environment variable: + +```bash +# Force refresh all caches during build +MARKDOWN_EXEC_CACHE_REFRESH=1 mkdocs build + +# Or with other truthy values +MARKDOWN_EXEC_CACHE_REFRESH=yes mkdocs build +MARKDOWN_EXEC_CACHE_REFRESH=true mkdocs build +MARKDOWN_EXEC_CACHE_REFRESH=on mkdocs build +``` + +This is useful for: +- CI/CD pipelines where you want fresh builds +- Ensuring all documentation is up-to-date +- Debugging cache-related issues + +## Clearing Cache + +### Delete Specific Cache Entry + +Remove the cache file for a specific custom ID: + +```bash +rm .markdown-exec-cache/my-custom-id.cache +``` + +### Clear All Cache + +Remove the entire cache directory: + +```bash +rm -rf .markdown-exec-cache/ +``` + +## How It Works + +1. **Hash Computation**: For `cache="yes"`, a SHA-256 hash is computed from: + - The code content + - Execution options (language, HTML mode, working directory, etc.) + +2. **Cache Lookup**: Before execution, the filesystem cache is checked for a matching entry + +3. **Execution & Storage**: If no cached result is found: + - Code is executed + - Output is stored in the filesystem cache + +4. **Cache Retrieval**: Cached output is used instead of re-executing the code + +## Best Practices + +### When to Use Caching + +✅ **Good use cases:** +- Generating plots, diagrams, or images +- Running expensive computations +- Calling external APIs or services +- Processing large datasets + +❌ **Avoid caching for:** +- Simple print statements +- Code demonstrating output variations +- Time-sensitive or non-deterministic code + +### Choosing Cache Type + +- **`cache="yes"`** (hash-based): + - Automatically invalidated when code changes + - Great for development and production + - No manual cache management needed + +- **`cache="custom-id"`** (custom ID): + - Use for expensive operations where you want explicit control + - Easier to identify and manage specific cache files + - Requires manual invalidation or `refresh="yes"` when code changes + +### Cache Invalidation Strategy + +**For hash-based caching (`cache="yes"`):** +- Cache is automatically invalidated when code or options change +- No manual intervention needed + +**For custom ID caching (`cache="custom-id"`):** + +1. **Change the ID** when you want to force re-execution: + ```markdown + cache="my-plot-v2" # Changed from my-plot + ``` + +2. **Use refresh temporarily**: + ```markdown + cache="my-plot" refresh="yes" # Remove refresh="yes" after update + ``` + +3. **Use global refresh** for all caches: + ```bash + MARKDOWN_EXEC_CACHE_REFRESH=1 mkdocs build + ``` + +4. **Clear cache directory** before important builds: + ```bash + rm -rf .markdown-exec-cache/ + ``` + +## Examples + +### Caching a Matplotlib Plot + +````markdown +```python exec="yes" html="yes" cache="population-chart" +import matplotlib.pyplot as plt +import io +import base64 + +# Expensive plot generation +fig, ax = plt.subplots() +ax.plot([1, 2, 3], [1, 4, 9]) +ax.set_title("Population Growth") + +# Save to base64 +buffer = io.BytesIO() +plt.savefig(buffer, format='png') +buffer.seek(0) +img_str = base64.b64encode(buffer.read()).decode() +print(f'') +plt.close() +``` +```` + +### Caching API Calls + +````markdown +```python exec="yes" cache="github-stars" refresh="no" +import requests +response = requests.get("https://api.github.com/repos/pawamoy/markdown-exec") +stars = response.json()["stargazers_count"] +print(f"⭐ **{stars}** stars on GitHub!") +``` +```` + +## Troubleshooting + +### Cache Not Working + +1. Ensure the cache directory is writable +2. Check that you're using `cache="yes"` or a custom ID +3. Verify the cache directory exists: `ls -la .markdown-exec-cache/` + +### Stale Cache Results + +1. Use `refresh="yes"` to force re-execution +2. Delete the specific cache file +3. Clear the entire cache directory + +### Large Cache Directory + +Cache files accumulate over time. Periodically clean up: + +```bash +# See cache directory size +du -sh .markdown-exec-cache/ + +# Remove all cache files +rm -rf .markdown-exec-cache/ +``` diff --git a/mkdocs.yml b/mkdocs.yml index dcc2b99..8ce3629 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -25,6 +25,7 @@ nav: - Pyodide: usage/pyodide.md - Shell: usage/shell.md - Tree: usage/tree.md + - Caching: usage/caching.md - Gallery: gallery.md - API reference: reference/api.md - Development: diff --git a/src/markdown_exec/__init__.py b/src/markdown_exec/__init__.py index bb4b647..e062e65 100644 --- a/src/markdown_exec/__init__.py +++ b/src/markdown_exec/__init__.py @@ -3,6 +3,7 @@ Utilities to execute code blocks in Markdown files. """ +from markdown_exec._internal.cache import CacheManager, get_cache_manager from markdown_exec._internal.formatters.base import ( ExecutionError, base_format, @@ -29,6 +30,7 @@ __all__ = [ "MARKDOWN_EXEC_AUTO", + "CacheManager", "ExecutionError", "HeadingReportingTreeprocessor", "IdPrependingTreeprocessor", @@ -43,6 +45,7 @@ "default_tabs", "formatter", "formatters", + "get_cache_manager", "get_logger", "markdown_config", "patch_loggers", diff --git a/src/markdown_exec/_internal/cache.py b/src/markdown_exec/_internal/cache.py new file mode 100644 index 0000000..3c6e42c --- /dev/null +++ b/src/markdown_exec/_internal/cache.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +from markdown_exec._internal.logger import get_logger + +_logger = get_logger(__name__) + + +def _get_project_root() -> Path: + """Determine the project root directory. + + Uses MKDOCS_CONFIG_DIR if available (set by MkDocs plugin), + otherwise falls back to current working directory. + + Returns: + Path to the project root directory. + """ + mkdocs_config_dir = os.getenv("MKDOCS_CONFIG_DIR") + if mkdocs_config_dir: + return Path(mkdocs_config_dir) + return Path.cwd() + + +class CacheManager: + """Manager for code execution caching. + + Provides filesystem-based caching for cross-build persistence. + """ + + def __init__(self, cache_dir: Path | None = None) -> None: + """Initialize the cache manager. + + Parameters: + cache_dir: Directory for filesystem cache. If None, uses .markdown-exec-cache + in the project root directory. + """ + if cache_dir is None: + cache_dir = _get_project_root() / ".markdown-exec-cache" + self.cache_dir = cache_dir + self.cache_dir.mkdir(parents=True, exist_ok=True) + + def _compute_hash(self, code: str, **options: Any) -> str: + """Compute a hash for the given code and options. + + Parameters: + code: The source code to hash. + **options: Additional options that affect execution (language, html, etc.). + + Returns: + A hex digest hash string. + """ + # Create a deterministic string from code and relevant options + # Exclude options that don't affect the output (like 'source', 'tabs', 'id', 'id_prefix') + relevant_options = { + k: v for k, v in sorted(options.items()) if k not in {"source", "tabs", "id", "id_prefix", "cache", "extra"} + } + + # Include 'extra' options that might affect execution + if "extra" in options and isinstance(options["extra"], dict): + relevant_options["extra"] = dict(sorted(options["extra"].items())) + + cache_key = json.dumps( + {"code": code, "options": relevant_options}, + sort_keys=True, + default=str, + ) + return hashlib.sha256(cache_key.encode()).hexdigest() + + def _get_cache_path(self, cache_id: str) -> Path: + """Get the filesystem path for a cache entry. + + Parameters: + cache_id: The cache identifier (hash or custom ID). + + Returns: + Path to the cache file. + """ + # Sanitize the cache_id to prevent path traversal + safe_id = "".join(c if c.isalnum() or c in "-_" else "_" for c in cache_id) + return self.cache_dir / f"{safe_id}.cache" + + def get( + self, + cache_id: str | None, + code: str, + refresh: bool = False, # noqa: FBT001, FBT002 + **options: Any, + ) -> str | None: + """Retrieve cached output for the given code. + + Parameters: + cache_id: Custom cache identifier, or None to use hash-based caching. + code: The source code. + refresh: If True, ignore cache and force re-execution. + **options: Execution options used for hash computation. + + Returns: + Cached output string, or None if not found or refresh is True. + """ + # Force cache miss if refresh is requested + if refresh: + _logger.debug("Cache refresh requested, forcing re-execution") + return None + + # Determine the cache key + cache_key = self._compute_hash(code, **options) if cache_id is None else cache_id + + # Check filesystem cache + cache_path = self._get_cache_path(cache_key) + if cache_path.exists(): + try: + output = cache_path.read_text(encoding="utf-8") + except OSError as error: + _logger.warning("Failed to read cache file %s: %s", cache_path, error) + else: + _logger.debug("Cache hit: %s", cache_key) + return output + + _logger.debug("Cache miss: %s", cache_key) + return None + + def set( + self, + cache_id: str | None, + code: str, + output: str, + **options: Any, + ) -> None: + """Store output in cache for the given code. + + Parameters: + cache_id: Custom cache identifier, or None to use hash-based caching. + code: The source code. + output: The execution output to cache. + **options: Execution options used for hash computation. + """ + # Determine the cache key + cache_key = self._compute_hash(code, **options) if cache_id is None else cache_id + + # Write to filesystem cache + cache_path = self._get_cache_path(cache_key) + try: + cache_path.write_text(output, encoding="utf-8") + _logger.debug("Cached to filesystem: %s (%s)", cache_key, cache_path) + except OSError as error: + _logger.warning("Failed to write cache file %s: %s", cache_path, error) + + def clear(self, cache_id: str | None = None) -> None: + """Clear the filesystem cache. + + Parameters: + cache_id: Specific cache ID to clear, or None to clear all. + """ + if cache_id is None: + # Clear all cache files + for cache_file in self.cache_dir.glob("*.cache"): + try: + cache_file.unlink() + _logger.debug("Deleted cache file: %s", cache_file) + except OSError as error: + _logger.warning("Failed to delete cache file %s: %s", cache_file, error) + else: + # Clear specific cache file + cache_path = self._get_cache_path(cache_id) + if cache_path.exists(): + try: + cache_path.unlink() + _logger.debug("Deleted cache file: %s", cache_path) + except OSError as error: + _logger.warning("Failed to delete cache file %s: %s", cache_path, error) + + +# Global cache manager instance +_cache_manager: CacheManager | None = None + + +def get_cache_manager() -> CacheManager: + """Get or create the global cache manager instance. + + Returns: + The global CacheManager instance. + """ + global _cache_manager # noqa: PLW0603 + if _cache_manager is None: + _cache_manager = CacheManager() + return _cache_manager diff --git a/src/markdown_exec/_internal/formatters/base.py b/src/markdown_exec/_internal/formatters/base.py index 00256cb..03927c0 100644 --- a/src/markdown_exec/_internal/formatters/base.py +++ b/src/markdown_exec/_internal/formatters/base.py @@ -10,6 +10,7 @@ from markupsafe import Markup +from markdown_exec._internal.cache import get_cache_manager from markdown_exec._internal.logger import get_logger from markdown_exec._internal.rendering import MarkdownConverter, add_source, code_block @@ -105,6 +106,8 @@ def base_format( update_toc: bool = True, workdir: str | None = None, width: int | None = None, + cache: bool | str = False, + refresh: bool = False, **options: Any, ) -> Markup: """Execute code and return HTML. @@ -128,6 +131,9 @@ def base_format( update_toc: Whether to include generated headings into the Markdown table of contents (toc extension). workdir: The working directory to use for the execution. + cache: Whether to enable caching. If True, uses hash-based caching. + If a string, uses that string as a custom cache ID for cross-build persistence. + refresh: If True, forces re-execution even if cached result exists. **options: Additional options passed from the formatter. Returns: @@ -142,20 +148,62 @@ def base_format( source_input = code source_output = code - try: - with working_directory(workdir), console_width(width): - output = run(source_input, returncode=returncode, session=session, id=id, **extra) - except ExecutionError as error: - identifier = id or extra.get("title", "") - identifier = identifier and f"'{identifier}' " - exit_message = "errors" if error.returncode is None else f"unexpected code {error.returncode}" - log_message = ( - f"Execution of {language} code block {identifier}exited with {exit_message}\n\n" - f"Code block is:\n\n{_format_log_details(source_input)}\n\n" - f"Output is:\n\n{_format_log_details(str(error), strip_fences=True)}\n" - ) - _logger.warning(log_message) - return markdown.convert(str(error)) + # Check cache if enabled + output = None + if cache: + cache_manager = get_cache_manager() + cache_id = cache if isinstance(cache, str) else None + + # Check for global refresh trigger via environment variable + global_refresh = os.getenv("MARKDOWN_EXEC_CACHE_REFRESH", "").lower() in {"1", "true", "yes", "on"} + effective_refresh = refresh or global_refresh + + # Build cache options (exclude cache itself and other non-execution options) + cache_options = { + "language": language, + "html": html, + "result": result, + "returncode": returncode, + "workdir": workdir, + "width": width, + "extra": extra, + } + + output = cache_manager.get(cache_id, source_input, refresh=effective_refresh, **cache_options) + if output is not None: + _logger.debug("Using cached output for code block") + + # Execute if not cached + if output is None: + try: + with working_directory(workdir), console_width(width): + output = run(source_input, returncode=returncode, session=session, id=id, **extra) + except ExecutionError as error: + identifier = id or extra.get("title", "") + identifier = identifier and f"'{identifier}' " + exit_message = "errors" if error.returncode is None else f"unexpected code {error.returncode}" + log_message = ( + f"Execution of {language} code block {identifier}exited with {exit_message}\n\n" + f"Code block is:\n\n{_format_log_details(source_input)}\n\n" + f"Output is:\n\n{_format_log_details(str(error), strip_fences=True)}\n" + ) + _logger.warning(log_message) + return markdown.convert(str(error)) + + # Cache the output if caching is enabled + if cache: + cache_manager = get_cache_manager() + cache_id = cache if isinstance(cache, str) else None + cache_options = { + "language": language, + "html": html, + "result": result, + "returncode": returncode, + "workdir": workdir, + "width": width, + "extra": extra, + } + cache_manager.set(cache_id, source_input, output, **cache_options) if not output and not source: return Markup() diff --git a/src/markdown_exec/_internal/main.py b/src/markdown_exec/_internal/main.py index 494980c..5f47fd3 100644 --- a/src/markdown_exec/_internal/main.py +++ b/src/markdown_exec/_internal/main.py @@ -76,6 +76,18 @@ def validator( tabs = tuple(_tabs_re.split(tabs_value, maxsplit=1)) workdir_value = inputs.pop("workdir", None) width_value = int(inputs.pop("width", "0")) + + # Handle cache option: can be boolean or custom string ID + cache_value = inputs.pop("cache", "") + cache_enabled = ( + _to_bool(cache_value) + if cache_value.lower() in {"yes", "on", "true", "1", "no", "off", "false", "0", ""} + else cache_value + ) + + # Handle refresh option to force cache invalidation + refresh_value = _to_bool(inputs.pop("refresh", "no")) + options["id"] = id_value options["id_prefix"] = id_prefix_value options["html"] = html_value @@ -87,6 +99,8 @@ def validator( options["tabs"] = tabs options["workdir"] = workdir_value options["width"] = width_value + options["cache"] = cache_enabled + options["refresh"] = refresh_value options["extra"] = inputs return True diff --git a/tests/conftest.py b/tests/conftest.py index c79722c..f49d7d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,39 @@ """Configuration for the pytest test suite.""" +import shutil +import tempfile +from collections.abc import Generator + import pytest from markdown import Markdown from markdown_exec import formatter, formatters, validator +from markdown_exec._internal import cache as cache_module + + +@pytest.fixture(autouse=True) +def _isolate_cache(monkeypatch: pytest.MonkeyPatch) -> Generator[None, None, None]: + """Isolate cache directory for each test. + + This ensures tests use a temporary cache directory that is + cleaned up after each test, preventing cache pollution between tests. + """ + # Create a temporary directory for cache + tmpdir = tempfile.mkdtemp(prefix="markdown-exec-test-cache-") + + # Set MKDOCS_CONFIG_DIR to point to the temp directory + monkeypatch.setenv("MKDOCS_CONFIG_DIR", tmpdir) + + # Reset the global cache manager to pick up the new directory + cache_module._cache_manager = None + + yield + + # Clean up the temporary cache directory + shutil.rmtree(tmpdir, ignore_errors=True) + + # Reset the cache manager again + cache_module._cache_manager = None @pytest.fixture diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..9697421 --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,423 @@ +"""Tests for the caching module.""" + +from __future__ import annotations + +import os +import tempfile +from pathlib import Path +from textwrap import dedent +from typing import TYPE_CHECKING + +import markdown_exec._internal.cache as cache_module +from markdown_exec._internal.cache import CacheManager, _get_project_root, get_cache_manager + +if TYPE_CHECKING: + from markdown import Markdown + + +def test_get_project_root_with_mkdocs_config_dir() -> None: + """Test project root detection with MKDOCS_CONFIG_DIR env var.""" + old_value = os.environ.get("MKDOCS_CONFIG_DIR") + try: + with tempfile.TemporaryDirectory() as test_dir: + os.environ["MKDOCS_CONFIG_DIR"] = test_dir + assert _get_project_root() == Path(test_dir) + finally: + if old_value is None: + os.environ.pop("MKDOCS_CONFIG_DIR", None) + else: + os.environ["MKDOCS_CONFIG_DIR"] = old_value + + +def test_get_project_root_without_mkdocs_config_dir() -> None: + """Test project root detection falls back to cwd.""" + old_value = os.environ.get("MKDOCS_CONFIG_DIR") + try: + os.environ.pop("MKDOCS_CONFIG_DIR", None) + assert _get_project_root() == Path.cwd() + finally: + if old_value is not None: + os.environ["MKDOCS_CONFIG_DIR"] = old_value + + +def test_cache_manager_default_uses_project_root() -> None: + """Test that CacheManager uses project root by default.""" + old_value = os.environ.get("MKDOCS_CONFIG_DIR") + try: + with tempfile.TemporaryDirectory() as tmpdir: + os.environ["MKDOCS_CONFIG_DIR"] = tmpdir + cache_manager = CacheManager() + expected_dir = Path(tmpdir) / ".markdown-exec-cache" + assert cache_manager.cache_dir == expected_dir + assert cache_manager.cache_dir.exists() + finally: + if old_value is None: + os.environ.pop("MKDOCS_CONFIG_DIR", None) + else: + os.environ["MKDOCS_CONFIG_DIR"] = old_value + + +def test_cache_manager_hash_based_filesystem() -> None: + """Test hash-based caching on filesystem.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache_manager = CacheManager(cache_dir=Path(tmpdir)) + cache_manager.clear() # Clear any existing cache + + code = "print('hello')" + output = "hello\n" + + # First get should return None + cached = cache_manager.get(None, code) + assert cached is None + + # Set cache + cache_manager.set(None, code, output) + + # Second get should return cached value + cached = cache_manager.get(None, code) + assert cached == output + + # Verify cache file exists + cache_files = list(Path(tmpdir).glob("*.cache")) + assert len(cache_files) == 1 + + +def test_cache_manager_custom_id_filesystem() -> None: + """Test custom ID caching on filesystem.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache_manager = CacheManager(cache_dir=Path(tmpdir)) + + code = "print('hello')" + output = "hello\n" + cache_id = "my-custom-id" + + # First get should return None + cached = cache_manager.get(cache_id, code) + assert cached is None + + # Set cache + cache_manager.set(cache_id, code, output) + + # Second get should return cached value + cached = cache_manager.get(cache_id, code) + assert cached == output + + # Verify cache file with custom ID exists + cache_path = cache_manager._get_cache_path(cache_id) + assert cache_path.exists() + assert cache_path.name == "my-custom-id.cache" + + +def test_cache_different_options_different_cache() -> None: + """Test that different options produce different cache entries.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache_manager = CacheManager(cache_dir=Path(tmpdir)) + cache_manager.clear() + + code = "print('hello')" + output1 = "hello\n" + output2 = "HELLO\n" + + # Cache with different options + cache_manager.set(None, code, output1, language="python") + cache_manager.set(None, code, output2, language="bash") + + # Should retrieve different outputs based on options + cached1 = cache_manager.get(None, code, language="python") + cached2 = cache_manager.get(None, code, language="bash") + + assert cached1 == output1 + assert cached2 == output2 + + +def test_cache_clear_filesystem() -> None: + """Test clearing filesystem cache.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache_manager = CacheManager(cache_dir=Path(tmpdir)) + + code = "print('hello')" + output = "hello\n" + cache_id = "test-id" + + cache_manager.set(cache_id, code, output) + assert cache_manager.get(cache_id, code) == output + + cache_manager.clear(cache_id) + assert cache_manager.get(cache_id, code) is None + + +def test_cache_clear_all_filesystem() -> None: + """Test clearing all caches.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache_manager = CacheManager(cache_dir=Path(tmpdir)) + + code = "print('hello')" + output = "hello\n" + + cache_manager.set(None, code, output) + assert cache_manager.get(None, code) == output + + cache_manager.clear() + assert cache_manager.get(None, code) is None + + +def test_get_cache_manager_singleton() -> None: + """Test that get_cache_manager returns the same instance.""" + manager1 = get_cache_manager() + manager2 = get_cache_manager() + assert manager1 is manager2 + + +def test_cache_integration_with_markdown(md: Markdown) -> None: + """Test caching with actual markdown execution (hash-based). + + Parameters: + md: A Markdown instance (fixture). + """ + # Clear cache before test + cache_manager = get_cache_manager() + cache_manager.clear() + + # First execution should run the code + html1 = md.convert( + dedent( + """ + ```python exec="yes" cache="yes" + print("**Bold!**") + ``` + """, + ), + ) + assert html1 == "

Bold!

" + + # Second execution should use cache (same code) + html2 = md.convert( + dedent( + """ + ```python exec="yes" cache="yes" + print("**Bold!**") + ``` + """, + ), + ) + assert html2 == "

Bold!

" + + # Different code should not use cache + html3 = md.convert( + dedent( + """ + ```python exec="yes" cache="yes" + print("**Different!**") + ``` + """, + ), + ) + assert html3 == "

Different!

" + + +def test_cache_integration_custom_id(md: Markdown) -> None: + """Test caching with custom ID. + + Parameters: + md: A Markdown instance (fixture). + """ + with tempfile.TemporaryDirectory() as tmpdir: + cache_manager = CacheManager(cache_dir=Path(tmpdir)) + # Replace global instance temporarily + old_manager = cache_module._cache_manager + cache_module._cache_manager = cache_manager + + try: + # First execution with custom ID + html1 = md.convert( + dedent( + """ + ```python exec="yes" cache="my-plot" + print("**Plot!**") + ``` + """, + ), + ) + assert html1 == "

Plot!

" + + # Verify cache file exists with custom ID + cache_path = Path(tmpdir) / "my-plot.cache" + assert cache_path.exists() + + # Second execution should use cache + html2 = md.convert( + dedent( + """ + ```python exec="yes" cache="my-plot" + print("**Plot!**") + ``` + """, + ), + ) + assert html2 == "

Plot!

" + finally: + # Restore original manager + cache_module._cache_manager = old_manager + + +def test_cache_disabled_by_default(md: Markdown) -> None: + """Test that caching is disabled by default. + + Parameters: + md: A Markdown instance (fixture). + """ + cache_manager = get_cache_manager() + cache_manager.clear() + + # Execute without cache option + html = md.convert( + dedent( + """ + ```python exec="yes" + print("**No cache!**") + ``` + """, + ), + ) + assert html == "

No cache!

" + + # Cache should be empty + # We can't directly check, but we can verify by executing with cache and seeing it's a miss + # This is implicitly tested by the fact that other tests need to explicitly enable cache + + +def test_cache_sanitizes_ids() -> None: + """Test that cache IDs are sanitized to prevent path traversal.""" + cache_manager = CacheManager() + + # Try various dangerous IDs + dangerous_ids = [ + "../../../etc/passwd", + "../../test", + "test/../../file", + "test/../file", + "test\\file", + ] + + for dangerous_id in dangerous_ids: + cache_path = cache_manager._get_cache_path(dangerous_id) + # Ensure the path is within the cache directory + assert cache_manager.cache_dir in cache_path.parents or cache_path.parent == cache_manager.cache_dir + # Ensure no directory separators in the filename + assert "/" not in cache_path.name + assert "\\" not in cache_path.name + + +def test_cache_refresh_forces_reexecution() -> None: + """Test that refresh parameter forces cache invalidation.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache_manager = CacheManager(cache_dir=Path(tmpdir)) + cache_manager.clear() + + code = "print('hello')" + output = "hello\n" + + # Cache some output + cache_manager.set(None, code, output) + + # Normal get should return cached value + cached = cache_manager.get(None, code, refresh=False) + assert cached == output + + # Get with refresh=True should return None (cache miss) + cached = cache_manager.get(None, code, refresh=True) + assert cached is None + + +def test_cache_refresh_integration(md: Markdown) -> None: + """Test refresh option in actual markdown execution. + + Parameters: + md: A Markdown instance (fixture). + """ + cache_manager = get_cache_manager() + cache_manager.clear() + + # First execution with cache + html1 = md.convert( + dedent( + """ + ```python exec="yes" cache="yes" + print("**First!**") + ``` + """, + ), + ) + assert html1 == "

First!

" + + # Second execution with cache should use cached result + html2 = md.convert( + dedent( + """ + ```python exec="yes" cache="yes" + print("**First!**") + ``` + """, + ), + ) + assert html2 == "

First!

" + + # Execution with refresh=yes should re-execute even with cache + # (Note: The actual output would be the same since the code is the same, + # but we verify the refresh parameter is accepted) + html3 = md.convert( + dedent( + """ + ```python exec="yes" cache="yes" refresh="yes" + print("**First!**") + ``` + """, + ), + ) + assert html3 == "

First!

" + + +def test_cache_global_refresh_env_var(md: Markdown) -> None: + """Test global refresh via MARKDOWN_EXEC_CACHE_REFRESH environment variable. + + Parameters: + md: A Markdown instance (fixture). + """ + cache_manager = get_cache_manager() + cache_manager.clear() + + # First execution with cache + html1 = md.convert( + dedent( + """ + ```python exec="yes" cache="yes" + print("**Cached!**") + ``` + """, + ), + ) + assert html1 == "

Cached!

" + + # Set global refresh environment variable + old_value = os.environ.get("MARKDOWN_EXEC_CACHE_REFRESH") + try: + os.environ["MARKDOWN_EXEC_CACHE_REFRESH"] = "1" + + # Execution with global refresh should re-execute even without refresh parameter + # (Note: The actual output would be the same since the code is the same, + # but we verify the global refresh is triggered) + html2 = md.convert( + dedent( + """ + ```python exec="yes" cache="yes" + print("**Cached!**") + ``` + """, + ), + ) + assert html2 == "

Cached!

" + finally: + if old_value is None: + os.environ.pop("MARKDOWN_EXEC_CACHE_REFRESH", None) + else: + os.environ["MARKDOWN_EXEC_CACHE_REFRESH"] = old_value From 91d8cab40aef0a014b712ad58a0d588baaf414bb Mon Sep 17 00:00:00 2001 From: John Lyu Date: Fri, 5 Dec 2025 11:28:09 +0800 Subject: [PATCH 2/4] fixup! feat: Add filesystem-only caching with global refresh support --- docs/usage/caching.md | 56 ++++++++++++------- .../_internal/formatters/base.py | 11 ---- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/docs/usage/caching.md b/docs/usage/caching.md index d9f36e3..7582a14 100644 --- a/docs/usage/caching.md +++ b/docs/usage/caching.md @@ -14,7 +14,7 @@ When generating images, charts, or running expensive computations in your docume All cached results are stored in `.markdown-exec-cache/` in your project root directory: -``` +```sh your-project/ ├── docs/ ├── mkdocs.yml @@ -62,19 +62,21 @@ The cache file will be stored as `.markdown-exec-cache/my-plot.cache`. To force re-execution and update the cache for a specific code block, use `refresh="yes"`: -```markdown +````markdown ```python exec="yes" cache="my-plot" refresh="yes" # This will always re-execute and update the cache print("Fresh execution!") ``` -``` +```` !!! note "refresh vs removing cache" - **`refresh="yes"`** forces re-execution but **keeps the cache enabled** - it updates the cached result for future builds. - - **Removing `cache` option** completely disables caching - the code executes every time with no caching at all. - - Use `refresh="yes"` when you want to update stale cache but keep caching benefits for subsequent builds. +**`refresh="yes"`** forces re-execution but **keeps the cache enabled** - it updates the cached result for future builds. + +```bash +**Removing `cache` option** completely disables caching - the code executes every time with no caching at all. + +Use `refresh="yes"` when you want to update stale cache but keep caching benefits for subsequent builds. +``` ### Global Cache Refresh @@ -91,6 +93,7 @@ MARKDOWN_EXEC_CACHE_REFRESH=on mkdocs build ``` This is useful for: + - CI/CD pipelines where you want fresh builds - Ensuring all documentation is up-to-date - Debugging cache-related issues @@ -116,28 +119,32 @@ rm -rf .markdown-exec-cache/ ## How It Works 1. **Hash Computation**: For `cache="yes"`, a SHA-256 hash is computed from: + - The code content - Execution options (language, HTML mode, working directory, etc.) - -2. **Cache Lookup**: Before execution, the filesystem cache is checked for a matching entry - -3. **Execution & Storage**: If no cached result is found: + +1. **Cache Lookup**: Before execution, the filesystem cache is checked for a matching entry + +1. **Execution & Storage**: If no cached result is found: + - Code is executed - Output is stored in the filesystem cache - -4. **Cache Retrieval**: Cached output is used instead of re-executing the code + +1. **Cache Retrieval**: Cached output is used instead of re-executing the code ## Best Practices ### When to Use Caching ✅ **Good use cases:** + - Generating plots, diagrams, or images - Running expensive computations - Calling external APIs or services - Processing large datasets ❌ **Avoid caching for:** + - Simple print statements - Code demonstrating output variations - Time-sensitive or non-deterministic code @@ -145,11 +152,13 @@ rm -rf .markdown-exec-cache/ ### Choosing Cache Type - **`cache="yes"`** (hash-based): + - Automatically invalidated when code changes - Great for development and production - No manual cache management needed - **`cache="custom-id"`** (custom ID): + - Use for expensive operations where you want explicit control - Easier to identify and manage specific cache files - Requires manual invalidation or `refresh="yes"` when code changes @@ -157,27 +166,32 @@ rm -rf .markdown-exec-cache/ ### Cache Invalidation Strategy **For hash-based caching (`cache="yes"`):** + - Cache is automatically invalidated when code or options change - No manual intervention needed **For custom ID caching (`cache="custom-id"`):** 1. **Change the ID** when you want to force re-execution: + ```markdown cache="my-plot-v2" # Changed from my-plot ``` -2. **Use refresh temporarily**: +1. **Use refresh temporarily**: + ```markdown cache="my-plot" refresh="yes" # Remove refresh="yes" after update ``` -3. **Use global refresh** for all caches: +1. **Use global refresh** for all caches: + ```bash MARKDOWN_EXEC_CACHE_REFRESH=1 mkdocs build ``` -4. **Clear cache directory** before important builds: +1. **Clear cache directory** before important builds: + ```bash rm -rf .markdown-exec-cache/ ``` @@ -223,14 +237,14 @@ print(f"⭐ **{stars}** stars on GitHub!") ### Cache Not Working 1. Ensure the cache directory is writable -2. Check that you're using `cache="yes"` or a custom ID -3. Verify the cache directory exists: `ls -la .markdown-exec-cache/` +1. Check that you're using `cache="yes"` or a custom ID +1. Verify the cache directory exists: `ls -la .markdown-exec-cache/` ### Stale Cache Results 1. Use `refresh="yes"` to force re-execution -2. Delete the specific cache file -3. Clear the entire cache directory +1. Delete the specific cache file +1. Clear the entire cache directory ### Large Cache Directory diff --git a/src/markdown_exec/_internal/formatters/base.py b/src/markdown_exec/_internal/formatters/base.py index 03927c0..da6843a 100644 --- a/src/markdown_exec/_internal/formatters/base.py +++ b/src/markdown_exec/_internal/formatters/base.py @@ -192,17 +192,6 @@ def base_format( # Cache the output if caching is enabled if cache: - cache_manager = get_cache_manager() - cache_id = cache if isinstance(cache, str) else None - cache_options = { - "language": language, - "html": html, - "result": result, - "returncode": returncode, - "workdir": workdir, - "width": width, - "extra": extra, - } cache_manager.set(cache_id, source_input, output, **cache_options) if not output and not source: From 0303b3f544ac4f50eafc0503afbd488373f5764c Mon Sep 17 00:00:00 2001 From: John Lyu Date: Fri, 6 Mar 2026 15:30:16 +0800 Subject: [PATCH 3/4] fixup! feat: Add filesystem-only caching with global refresh support --- docs/usage/caching.md | 23 +--- src/markdown_exec/_internal/cache.py | 36 +++++- .../_internal/formatters/base.py | 8 +- src/markdown_exec/_internal/main.py | 4 - src/markdown_exec/_internal/mkdocs_plugin.py | 13 +- tests/test_cache.py | 116 +++++++++--------- 6 files changed, 107 insertions(+), 93 deletions(-) diff --git a/docs/usage/caching.md b/docs/usage/caching.md index 7582a14..046424f 100644 --- a/docs/usage/caching.md +++ b/docs/usage/caching.md @@ -60,27 +60,12 @@ The cache file will be stored as `.markdown-exec-cache/my-plot.cache`. ### Cache Invalidation -To force re-execution and update the cache for a specific code block, use `refresh="yes"`: +The cache is automatically invalidated when the code content or execution options change (a new hash is computed). **Stale cache files** — from code blocks that have been removed or changed — are cleaned up automatically: -````markdown -```python exec="yes" cache="my-plot" refresh="yes" -# This will always re-execute and update the cache -print("Fresh execution!") -``` -```` - -!!! note "refresh vs removing cache" -**`refresh="yes"`** forces re-execution but **keeps the cache enabled** - it updates the cached result for future builds. - -```bash -**Removing `cache` option** completely disables caching - the code executes every time with no caching at all. - -Use `refresh="yes"` when you want to update stale cache but keep caching benefits for subsequent builds. -``` - -### Global Cache Refresh +- **MkDocs builds**: at the end of each build (`on_post_build`), any `.cache` file not used during that build is deleted. +- **Standalone usage**: stale files are cleaned up when the Python process exits. -To refresh **all** cached results at once, set the `MARKDOWN_EXEC_CACHE_REFRESH` environment variable: +To force re-execution of **all** cached blocks (e.g. when an external dependency changes), use the `MARKDOWN_EXEC_CACHE_REFRESH` environment variable instead of touching the code: ```bash # Force refresh all caches during build diff --git a/src/markdown_exec/_internal/cache.py b/src/markdown_exec/_internal/cache.py index 3c6e42c..eca2a73 100644 --- a/src/markdown_exec/_internal/cache.py +++ b/src/markdown_exec/_internal/cache.py @@ -1,5 +1,6 @@ from __future__ import annotations +import atexit import hashlib import json import os @@ -43,6 +44,7 @@ def __init__(self, cache_dir: Path | None = None) -> None: cache_dir = _get_project_root() / ".markdown-exec-cache" self.cache_dir = cache_dir self.cache_dir.mkdir(parents=True, exist_ok=True) + self._current_hashes: set[str] = set() def _compute_hash(self, code: str, **options: Any) -> str: """Compute a hash for the given code and options. @@ -88,7 +90,6 @@ def get( self, cache_id: str | None, code: str, - refresh: bool = False, # noqa: FBT001, FBT002 **options: Any, ) -> str | None: """Retrieve cached output for the given code. @@ -96,15 +97,14 @@ def get( Parameters: cache_id: Custom cache identifier, or None to use hash-based caching. code: The source code. - refresh: If True, ignore cache and force re-execution. **options: Execution options used for hash computation. Returns: - Cached output string, or None if not found or refresh is True. + Cached output string, or None if not found. """ - # Force cache miss if refresh is requested - if refresh: - _logger.debug("Cache refresh requested, forcing re-execution") + # Check for global refresh trigger via environment variable + if os.getenv("MARKDOWN_EXEC_CACHE_REFRESH", "").lower() in {"1", "true", "yes", "on"}: + _logger.debug("Global cache refresh active, forcing re-execution") return None # Determine the cache key @@ -119,6 +119,7 @@ def get( _logger.warning("Failed to read cache file %s: %s", cache_path, error) else: _logger.debug("Cache hit: %s", cache_key) + self._current_hashes.add(cache_key) return output _logger.debug("Cache miss: %s", cache_key) @@ -149,6 +150,24 @@ def set( _logger.debug("Cached to filesystem: %s (%s)", cache_key, cache_path) except OSError as error: _logger.warning("Failed to write cache file %s: %s", cache_path, error) + self._current_hashes.add(cache_key) + + def cleanup_stale(self) -> None: + """Delete cache files that were not used in the current build session. + + Compares all `.cache` files in the cache directory against + the hashes collected during this session and removes any that + are no longer referenced. + """ + if not self.cache_dir.exists(): + return + for cache_file in self.cache_dir.glob("*.cache"): + if cache_file.stem not in self._current_hashes: + try: + cache_file.unlink() + _logger.debug("Deleted stale cache file: %s", cache_file) + except OSError as error: + _logger.warning("Failed to delete stale cache file %s: %s", cache_file, error) def clear(self, cache_id: str | None = None) -> None: """Clear the filesystem cache. @@ -188,4 +207,9 @@ def get_cache_manager() -> CacheManager: global _cache_manager # noqa: PLW0603 if _cache_manager is None: _cache_manager = CacheManager() + # For standalone (non-MkDocs) usage, automatically clean up stale + # cache files when the process exits. In MkDocs context the plugin + # hook handles this, so we skip registration there. + if not os.getenv("MKDOCS_CONFIG_DIR"): + atexit.register(_cache_manager.cleanup_stale) return _cache_manager diff --git a/src/markdown_exec/_internal/formatters/base.py b/src/markdown_exec/_internal/formatters/base.py index da6843a..a64b1e5 100644 --- a/src/markdown_exec/_internal/formatters/base.py +++ b/src/markdown_exec/_internal/formatters/base.py @@ -107,7 +107,6 @@ def base_format( workdir: str | None = None, width: int | None = None, cache: bool | str = False, - refresh: bool = False, **options: Any, ) -> Markup: """Execute code and return HTML. @@ -133,7 +132,6 @@ def base_format( workdir: The working directory to use for the execution. cache: Whether to enable caching. If True, uses hash-based caching. If a string, uses that string as a custom cache ID for cross-build persistence. - refresh: If True, forces re-execution even if cached result exists. **options: Additional options passed from the formatter. Returns: @@ -154,10 +152,6 @@ def base_format( cache_manager = get_cache_manager() cache_id = cache if isinstance(cache, str) else None - # Check for global refresh trigger via environment variable - global_refresh = os.getenv("MARKDOWN_EXEC_CACHE_REFRESH", "").lower() in {"1", "true", "yes", "on"} - effective_refresh = refresh or global_refresh - # Build cache options (exclude cache itself and other non-execution options) cache_options = { "language": language, @@ -169,7 +163,7 @@ def base_format( "extra": extra, } - output = cache_manager.get(cache_id, source_input, refresh=effective_refresh, **cache_options) + output = cache_manager.get(cache_id, source_input, **cache_options) if output is not None: _logger.debug("Using cached output for code block") diff --git a/src/markdown_exec/_internal/main.py b/src/markdown_exec/_internal/main.py index 5f47fd3..b9e918c 100644 --- a/src/markdown_exec/_internal/main.py +++ b/src/markdown_exec/_internal/main.py @@ -85,9 +85,6 @@ def validator( else cache_value ) - # Handle refresh option to force cache invalidation - refresh_value = _to_bool(inputs.pop("refresh", "no")) - options["id"] = id_value options["id_prefix"] = id_prefix_value options["html"] = html_value @@ -100,7 +97,6 @@ def validator( options["workdir"] = workdir_value options["width"] = width_value options["cache"] = cache_enabled - options["refresh"] = refresh_value options["extra"] = inputs return True diff --git a/src/markdown_exec/_internal/mkdocs_plugin.py b/src/markdown_exec/_internal/mkdocs_plugin.py index 4b7a11c..907ba38 100644 --- a/src/markdown_exec/_internal/mkdocs_plugin.py +++ b/src/markdown_exec/_internal/mkdocs_plugin.py @@ -13,6 +13,7 @@ from mkdocs.plugins import BasePlugin from mkdocs.utils import write_file +from markdown_exec._internal.cache import get_cache_manager from markdown_exec._internal.logger import patch_loggers from markdown_exec._internal.main import formatter, formatters, validator from markdown_exec._internal.rendering import MarkdownConverter, markdown_config @@ -106,6 +107,15 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: markdown_config.save(config.markdown_extensions, config.mdx_configs) return config + def on_pre_build(self, *, config: MkDocsConfig) -> None: # noqa: ARG002 + """Reset cache tracking state for a new build. + + Clears the set of hashes seen so far, then removes any cache files + left over from a previous build that are no longer referenced. + """ + mgr = get_cache_manager() + mgr._current_hashes = set() + def on_env( self, env: Environment, @@ -122,7 +132,8 @@ def on_env( return env def on_post_build(self, *, config: MkDocsConfig) -> None: # noqa: ARG002 - """Reset the plugin state.""" + """Clean up stale cache files and reset the plugin state.""" + get_cache_manager().cleanup_stale() MarkdownConverter.counter = 0 markdown_config.reset() if self.mkdocs_config_dir is None: diff --git a/tests/test_cache.py b/tests/test_cache.py index 9697421..d4f93ac 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -9,7 +9,11 @@ from typing import TYPE_CHECKING import markdown_exec._internal.cache as cache_module -from markdown_exec._internal.cache import CacheManager, _get_project_root, get_cache_manager +from markdown_exec._internal.cache import ( + CacheManager, + _get_project_root, + get_cache_manager, +) if TYPE_CHECKING: from markdown import Markdown @@ -308,73 +312,73 @@ def test_cache_sanitizes_ids() -> None: assert "\\" not in cache_path.name -def test_cache_refresh_forces_reexecution() -> None: - """Test that refresh parameter forces cache invalidation.""" +def test_cleanup_stale_removes_unreferenced_files() -> None: + """Test that cleanup_stale removes cache files not used in the current session.""" with tempfile.TemporaryDirectory() as tmpdir: cache_manager = CacheManager(cache_dir=Path(tmpdir)) - cache_manager.clear() - code = "print('hello')" - output = "hello\n" + # Create two cache entries + cache_manager.set(None, "print('a')", "a\n", language="python") + cache_manager.set(None, "print('b')", "b\n", language="python") + assert len(list(Path(tmpdir).glob("*.cache"))) == 2 - # Cache some output - cache_manager.set(None, code, output) + # Start a fresh session: only reference the first entry + cache_manager._current_hashes = set() + cache_manager.get(None, "print('a')", language="python") - # Normal get should return cached value - cached = cache_manager.get(None, code, refresh=False) - assert cached == output + # cleanup_stale should remove the second file + cache_manager.cleanup_stale() + remaining = list(Path(tmpdir).glob("*.cache")) + assert len(remaining) == 1 - # Get with refresh=True should return None (cache miss) - cached = cache_manager.get(None, code, refresh=True) - assert cached is None +def test_cleanup_stale_removes_custom_id_files() -> None: + """Test that cleanup_stale works with custom-ID cache entries.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache_manager = CacheManager(cache_dir=Path(tmpdir)) -def test_cache_refresh_integration(md: Markdown) -> None: - """Test refresh option in actual markdown execution. + cache_manager.set("keep-me", "print('keep')", "keep\n") + cache_manager.set("drop-me", "print('drop')", "drop\n") + assert len(list(Path(tmpdir).glob("*.cache"))) == 2 - Parameters: - md: A Markdown instance (fixture). - """ - cache_manager = get_cache_manager() - cache_manager.clear() + # New session: only the first ID is used + cache_manager._current_hashes = set() + cache_manager.get("keep-me", "print('keep')") - # First execution with cache - html1 = md.convert( - dedent( - """ - ```python exec="yes" cache="yes" - print("**First!**") - ``` - """, - ), - ) - assert html1 == "

First!

" + cache_manager.cleanup_stale() + remaining = list(Path(tmpdir).glob("*.cache")) + assert len(remaining) == 1 + assert remaining[0].stem == "keep-me" - # Second execution with cache should use cached result - html2 = md.convert( - dedent( - """ - ```python exec="yes" cache="yes" - print("**First!**") - ``` - """, - ), - ) - assert html2 == "

First!

" - # Execution with refresh=yes should re-execute even with cache - # (Note: The actual output would be the same since the code is the same, - # but we verify the refresh parameter is accepted) - html3 = md.convert( - dedent( - """ - ```python exec="yes" cache="yes" refresh="yes" - print("**First!**") - ``` - """, - ), - ) - assert html3 == "

First!

" +def test_current_hashes_tracked_on_set() -> None: + """Test that set() registers the hash in _current_hashes.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache_manager = CacheManager(cache_dir=Path(tmpdir)) + assert len(cache_manager._current_hashes) == 0 + + cache_manager.set(None, "print('x')", "x\n", language="python") + assert len(cache_manager._current_hashes) == 1 + + +def test_current_hashes_tracked_on_get_hit() -> None: + """Test that get() registers the hash in _current_hashes on a cache hit.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache_manager = CacheManager(cache_dir=Path(tmpdir)) + cache_manager.set(None, "print('x')", "x\n", language="python") + + cache_manager._current_hashes = set() + result = cache_manager.get(None, "print('x')", language="python") + assert result == "x\n" + assert len(cache_manager._current_hashes) == 1 + + +def test_cleanup_stale_noop_on_missing_dir() -> None: + """Test that cleanup_stale does not raise when cache dir has been removed.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache_manager = CacheManager(cache_dir=Path(tmpdir)) + # tmpdir is now deleted; cleanup_stale should be a no-op + cache_manager.cleanup_stale() # must not raise def test_cache_global_refresh_env_var(md: Markdown) -> None: From 8a4179c13add9eb83cfeb541c86029dbbf341495 Mon Sep 17 00:00:00 2001 From: John Lyu Date: Fri, 6 Mar 2026 15:42:06 +0800 Subject: [PATCH 4/4] fixup! feat: Add filesystem-only caching with global refresh support --- docs/usage/caching.md | 83 +--------- src/markdown_exec/_internal/cache.py | 49 ++---- .../_internal/formatters/base.py | 11 +- src/markdown_exec/_internal/main.py | 7 +- tests/test_cache.py | 154 +++--------------- 5 files changed, 45 insertions(+), 259 deletions(-) diff --git a/docs/usage/caching.md b/docs/usage/caching.md index 046424f..08a432e 100644 --- a/docs/usage/caching.md +++ b/docs/usage/caching.md @@ -19,7 +19,6 @@ your-project/ ├── docs/ ├── mkdocs.yml └── .markdown-exec-cache/ - ├── my-plot.cache # Custom ID cache └── abc123def456.cache # Hash-based cache files ``` @@ -44,20 +43,6 @@ print(f"Executed at: {time.time()}") The cache is automatically invalidated when the code or execution options change. -### Custom Cache IDs - -For more control, use a custom cache ID (string value). This is useful for expensive operations where you want explicit control over cache invalidation: - -````md exec="1" source="tabbed-left" tabs="Markdown|Rendered" -```python exec="yes" cache="my-plot" -import matplotlib.pyplot as plt -# Expensive plot generation... -print("Generated plot") -``` -```` - -The cache file will be stored as `.markdown-exec-cache/my-plot.cache`. - ### Cache Invalidation The cache is automatically invalidated when the code content or execution options change (a new hash is computed). **Stale cache files** — from code blocks that have been removed or changed — are cleaned up automatically: @@ -85,16 +70,6 @@ This is useful for: ## Clearing Cache -### Delete Specific Cache Entry - -Remove the cache file for a specific custom ID: - -```bash -rm .markdown-exec-cache/my-custom-id.cache -``` - -### Clear All Cache - Remove the entire cache directory: ```bash @@ -136,57 +111,24 @@ rm -rf .markdown-exec-cache/ ### Choosing Cache Type -- **`cache="yes"`** (hash-based): - - - Automatically invalidated when code changes - - Great for development and production - - No manual cache management needed - -- **`cache="custom-id"`** (custom ID): - - - Use for expensive operations where you want explicit control - - Easier to identify and manage specific cache files - - Requires manual invalidation or `refresh="yes"` when code changes +Use `cache="yes"` for all caching needs. The cache is automatically invalidated when the code or execution options change — no manual cache management needed. ### Cache Invalidation Strategy -**For hash-based caching (`cache="yes"`):** +Cache is automatically invalidated when code or options change — no manual intervention needed. To force re-execution of all cached blocks, use: -- Cache is automatically invalidated when code or options change -- No manual intervention needed - -**For custom ID caching (`cache="custom-id"`):** - -1. **Change the ID** when you want to force re-execution: - - ```markdown - cache="my-plot-v2" # Changed from my-plot - ``` - -1. **Use refresh temporarily**: - - ```markdown - cache="my-plot" refresh="yes" # Remove refresh="yes" after update - ``` - -1. **Use global refresh** for all caches: - - ```bash - MARKDOWN_EXEC_CACHE_REFRESH=1 mkdocs build - ``` - -1. **Clear cache directory** before important builds: +```bash +MARKDOWN_EXEC_CACHE_REFRESH=1 mkdocs build +``` - ```bash - rm -rf .markdown-exec-cache/ - ``` +Or [clear the cache directory](#clearing-cache) before the build. ## Examples ### Caching a Matplotlib Plot ````markdown -```python exec="yes" html="yes" cache="population-chart" +```python exec="yes" html="yes" cache="yes" import matplotlib.pyplot as plt import io import base64 @@ -206,17 +148,6 @@ plt.close() ``` ```` -### Caching API Calls - -````markdown -```python exec="yes" cache="github-stars" refresh="no" -import requests -response = requests.get("https://api.github.com/repos/pawamoy/markdown-exec") -stars = response.json()["stargazers_count"] -print(f"⭐ **{stars}** stars on GitHub!") -``` -```` - ## Troubleshooting ### Cache Not Working diff --git a/src/markdown_exec/_internal/cache.py b/src/markdown_exec/_internal/cache.py index eca2a73..25cb77b 100644 --- a/src/markdown_exec/_internal/cache.py +++ b/src/markdown_exec/_internal/cache.py @@ -73,29 +73,25 @@ def _compute_hash(self, code: str, **options: Any) -> str: ) return hashlib.sha256(cache_key.encode()).hexdigest() - def _get_cache_path(self, cache_id: str) -> Path: + def _get_cache_path(self, cache_key: str) -> Path: """Get the filesystem path for a cache entry. Parameters: - cache_id: The cache identifier (hash or custom ID). + cache_key: The cache key (hash). Returns: Path to the cache file. """ - # Sanitize the cache_id to prevent path traversal - safe_id = "".join(c if c.isalnum() or c in "-_" else "_" for c in cache_id) - return self.cache_dir / f"{safe_id}.cache" + return self.cache_dir / f"{cache_key}.cache" def get( self, - cache_id: str | None, code: str, **options: Any, ) -> str | None: """Retrieve cached output for the given code. Parameters: - cache_id: Custom cache identifier, or None to use hash-based caching. code: The source code. **options: Execution options used for hash computation. @@ -107,8 +103,7 @@ def get( _logger.debug("Global cache refresh active, forcing re-execution") return None - # Determine the cache key - cache_key = self._compute_hash(code, **options) if cache_id is None else cache_id + cache_key = self._compute_hash(code, **options) # Check filesystem cache cache_path = self._get_cache_path(cache_key) @@ -127,7 +122,6 @@ def get( def set( self, - cache_id: str | None, code: str, output: str, **options: Any, @@ -135,13 +129,11 @@ def set( """Store output in cache for the given code. Parameters: - cache_id: Custom cache identifier, or None to use hash-based caching. code: The source code. output: The execution output to cache. **options: Execution options used for hash computation. """ - # Determine the cache key - cache_key = self._compute_hash(code, **options) if cache_id is None else cache_id + cache_key = self._compute_hash(code, **options) # Write to filesystem cache cache_path = self._get_cache_path(cache_key) @@ -169,29 +161,14 @@ def cleanup_stale(self) -> None: except OSError as error: _logger.warning("Failed to delete stale cache file %s: %s", cache_file, error) - def clear(self, cache_id: str | None = None) -> None: - """Clear the filesystem cache. - - Parameters: - cache_id: Specific cache ID to clear, or None to clear all. - """ - if cache_id is None: - # Clear all cache files - for cache_file in self.cache_dir.glob("*.cache"): - try: - cache_file.unlink() - _logger.debug("Deleted cache file: %s", cache_file) - except OSError as error: - _logger.warning("Failed to delete cache file %s: %s", cache_file, error) - else: - # Clear specific cache file - cache_path = self._get_cache_path(cache_id) - if cache_path.exists(): - try: - cache_path.unlink() - _logger.debug("Deleted cache file: %s", cache_path) - except OSError as error: - _logger.warning("Failed to delete cache file %s: %s", cache_path, error) + def clear(self) -> None: + """Clear all cached files.""" + for cache_file in self.cache_dir.glob("*.cache"): + try: + cache_file.unlink() + _logger.debug("Deleted cache file: %s", cache_file) + except OSError as error: + _logger.warning("Failed to delete cache file %s: %s", cache_file, error) # Global cache manager instance diff --git a/src/markdown_exec/_internal/formatters/base.py b/src/markdown_exec/_internal/formatters/base.py index a64b1e5..003838a 100644 --- a/src/markdown_exec/_internal/formatters/base.py +++ b/src/markdown_exec/_internal/formatters/base.py @@ -106,7 +106,7 @@ def base_format( update_toc: bool = True, workdir: str | None = None, width: int | None = None, - cache: bool | str = False, + cache: bool = False, **options: Any, ) -> Markup: """Execute code and return HTML. @@ -130,8 +130,7 @@ def base_format( update_toc: Whether to include generated headings into the Markdown table of contents (toc extension). workdir: The working directory to use for the execution. - cache: Whether to enable caching. If True, uses hash-based caching. - If a string, uses that string as a custom cache ID for cross-build persistence. + cache: Whether to enable caching. **options: Additional options passed from the formatter. Returns: @@ -150,8 +149,6 @@ def base_format( output = None if cache: cache_manager = get_cache_manager() - cache_id = cache if isinstance(cache, str) else None - # Build cache options (exclude cache itself and other non-execution options) cache_options = { "language": language, @@ -163,7 +160,7 @@ def base_format( "extra": extra, } - output = cache_manager.get(cache_id, source_input, **cache_options) + output = cache_manager.get(source_input, **cache_options) if output is not None: _logger.debug("Using cached output for code block") @@ -186,7 +183,7 @@ def base_format( # Cache the output if caching is enabled if cache: - cache_manager.set(cache_id, source_input, output, **cache_options) + cache_manager.set(source_input, output, **cache_options) if not output and not source: return Markup() diff --git a/src/markdown_exec/_internal/main.py b/src/markdown_exec/_internal/main.py index b9e918c..2746660 100644 --- a/src/markdown_exec/_internal/main.py +++ b/src/markdown_exec/_internal/main.py @@ -77,13 +77,8 @@ def validator( workdir_value = inputs.pop("workdir", None) width_value = int(inputs.pop("width", "0")) - # Handle cache option: can be boolean or custom string ID cache_value = inputs.pop("cache", "") - cache_enabled = ( - _to_bool(cache_value) - if cache_value.lower() in {"yes", "on", "true", "1", "no", "off", "false", "0", ""} - else cache_value - ) + cache_enabled = _to_bool(cache_value) options["id"] = id_value options["id_prefix"] = id_prefix_value diff --git a/tests/test_cache.py b/tests/test_cache.py index d4f93ac..cb0ba14 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -8,7 +8,6 @@ from textwrap import dedent from typing import TYPE_CHECKING -import markdown_exec._internal.cache as cache_module from markdown_exec._internal.cache import ( CacheManager, _get_project_root, @@ -71,14 +70,14 @@ def test_cache_manager_hash_based_filesystem() -> None: output = "hello\n" # First get should return None - cached = cache_manager.get(None, code) + cached = cache_manager.get(code) assert cached is None # Set cache - cache_manager.set(None, code, output) + cache_manager.set(code, output) # Second get should return cached value - cached = cache_manager.get(None, code) + cached = cache_manager.get(code) assert cached == output # Verify cache file exists @@ -86,32 +85,6 @@ def test_cache_manager_hash_based_filesystem() -> None: assert len(cache_files) == 1 -def test_cache_manager_custom_id_filesystem() -> None: - """Test custom ID caching on filesystem.""" - with tempfile.TemporaryDirectory() as tmpdir: - cache_manager = CacheManager(cache_dir=Path(tmpdir)) - - code = "print('hello')" - output = "hello\n" - cache_id = "my-custom-id" - - # First get should return None - cached = cache_manager.get(cache_id, code) - assert cached is None - - # Set cache - cache_manager.set(cache_id, code, output) - - # Second get should return cached value - cached = cache_manager.get(cache_id, code) - assert cached == output - - # Verify cache file with custom ID exists - cache_path = cache_manager._get_cache_path(cache_id) - assert cache_path.exists() - assert cache_path.name == "my-custom-id.cache" - - def test_cache_different_options_different_cache() -> None: """Test that different options produce different cache entries.""" with tempfile.TemporaryDirectory() as tmpdir: @@ -123,12 +96,12 @@ def test_cache_different_options_different_cache() -> None: output2 = "HELLO\n" # Cache with different options - cache_manager.set(None, code, output1, language="python") - cache_manager.set(None, code, output2, language="bash") + cache_manager.set(code, output1, language="python") + cache_manager.set(code, output2, language="bash") # Should retrieve different outputs based on options - cached1 = cache_manager.get(None, code, language="python") - cached2 = cache_manager.get(None, code, language="bash") + cached1 = cache_manager.get(code, language="python") + cached2 = cache_manager.get(code, language="bash") assert cached1 == output1 assert cached2 == output2 @@ -141,13 +114,12 @@ def test_cache_clear_filesystem() -> None: code = "print('hello')" output = "hello\n" - cache_id = "test-id" - cache_manager.set(cache_id, code, output) - assert cache_manager.get(cache_id, code) == output + cache_manager.set(code, output) + assert cache_manager.get(code) == output - cache_manager.clear(cache_id) - assert cache_manager.get(cache_id, code) is None + cache_manager.clear() + assert cache_manager.get(code) is None def test_cache_clear_all_filesystem() -> None: @@ -158,11 +130,11 @@ def test_cache_clear_all_filesystem() -> None: code = "print('hello')" output = "hello\n" - cache_manager.set(None, code, output) - assert cache_manager.get(None, code) == output + cache_manager.set(code, output) + assert cache_manager.get(code) == output cache_manager.clear() - assert cache_manager.get(None, code) is None + assert cache_manager.get(code) is None def test_get_cache_manager_singleton() -> None: @@ -219,51 +191,6 @@ def test_cache_integration_with_markdown(md: Markdown) -> None: assert html3 == "

Different!

" -def test_cache_integration_custom_id(md: Markdown) -> None: - """Test caching with custom ID. - - Parameters: - md: A Markdown instance (fixture). - """ - with tempfile.TemporaryDirectory() as tmpdir: - cache_manager = CacheManager(cache_dir=Path(tmpdir)) - # Replace global instance temporarily - old_manager = cache_module._cache_manager - cache_module._cache_manager = cache_manager - - try: - # First execution with custom ID - html1 = md.convert( - dedent( - """ - ```python exec="yes" cache="my-plot" - print("**Plot!**") - ``` - """, - ), - ) - assert html1 == "

Plot!

" - - # Verify cache file exists with custom ID - cache_path = Path(tmpdir) / "my-plot.cache" - assert cache_path.exists() - - # Second execution should use cache - html2 = md.convert( - dedent( - """ - ```python exec="yes" cache="my-plot" - print("**Plot!**") - ``` - """, - ), - ) - assert html2 == "

Plot!

" - finally: - # Restore original manager - cache_module._cache_manager = old_manager - - def test_cache_disabled_by_default(md: Markdown) -> None: """Test that caching is disabled by default. @@ -290,41 +217,19 @@ def test_cache_disabled_by_default(md: Markdown) -> None: # This is implicitly tested by the fact that other tests need to explicitly enable cache -def test_cache_sanitizes_ids() -> None: - """Test that cache IDs are sanitized to prevent path traversal.""" - cache_manager = CacheManager() - - # Try various dangerous IDs - dangerous_ids = [ - "../../../etc/passwd", - "../../test", - "test/../../file", - "test/../file", - "test\\file", - ] - - for dangerous_id in dangerous_ids: - cache_path = cache_manager._get_cache_path(dangerous_id) - # Ensure the path is within the cache directory - assert cache_manager.cache_dir in cache_path.parents or cache_path.parent == cache_manager.cache_dir - # Ensure no directory separators in the filename - assert "/" not in cache_path.name - assert "\\" not in cache_path.name - - def test_cleanup_stale_removes_unreferenced_files() -> None: """Test that cleanup_stale removes cache files not used in the current session.""" with tempfile.TemporaryDirectory() as tmpdir: cache_manager = CacheManager(cache_dir=Path(tmpdir)) # Create two cache entries - cache_manager.set(None, "print('a')", "a\n", language="python") - cache_manager.set(None, "print('b')", "b\n", language="python") + cache_manager.set("print('a')", "a\n", language="python") + cache_manager.set("print('b')", "b\n", language="python") assert len(list(Path(tmpdir).glob("*.cache"))) == 2 # Start a fresh session: only reference the first entry cache_manager._current_hashes = set() - cache_manager.get(None, "print('a')", language="python") + cache_manager.get("print('a')", language="python") # cleanup_stale should remove the second file cache_manager.cleanup_stale() @@ -332,32 +237,13 @@ def test_cleanup_stale_removes_unreferenced_files() -> None: assert len(remaining) == 1 -def test_cleanup_stale_removes_custom_id_files() -> None: - """Test that cleanup_stale works with custom-ID cache entries.""" - with tempfile.TemporaryDirectory() as tmpdir: - cache_manager = CacheManager(cache_dir=Path(tmpdir)) - - cache_manager.set("keep-me", "print('keep')", "keep\n") - cache_manager.set("drop-me", "print('drop')", "drop\n") - assert len(list(Path(tmpdir).glob("*.cache"))) == 2 - - # New session: only the first ID is used - cache_manager._current_hashes = set() - cache_manager.get("keep-me", "print('keep')") - - cache_manager.cleanup_stale() - remaining = list(Path(tmpdir).glob("*.cache")) - assert len(remaining) == 1 - assert remaining[0].stem == "keep-me" - - def test_current_hashes_tracked_on_set() -> None: """Test that set() registers the hash in _current_hashes.""" with tempfile.TemporaryDirectory() as tmpdir: cache_manager = CacheManager(cache_dir=Path(tmpdir)) assert len(cache_manager._current_hashes) == 0 - cache_manager.set(None, "print('x')", "x\n", language="python") + cache_manager.set("print('x')", "x\n", language="python") assert len(cache_manager._current_hashes) == 1 @@ -365,10 +251,10 @@ def test_current_hashes_tracked_on_get_hit() -> None: """Test that get() registers the hash in _current_hashes on a cache hit.""" with tempfile.TemporaryDirectory() as tmpdir: cache_manager = CacheManager(cache_dir=Path(tmpdir)) - cache_manager.set(None, "print('x')", "x\n", language="python") + cache_manager.set("print('x')", "x\n", language="python") cache_manager._current_hashes = set() - result = cache_manager.get(None, "print('x')", language="python") + result = cache_manager.get("print('x')", language="python") assert result == "x\n" assert len(cache_manager._current_hashes) == 1