save runbook snapshot

Author: danielgafni

docs/.mkdocs-metaxy/src/mkdocs_metaxy/examples/core.py

@@ -4,6 +4,7 @@
 - Loading runbooks from .example.yaml files
 - Applying patches to show code evolution
 - Reading source files at different stages
+- Loading saved execution results from .example.result.json files
 """
 
 from __future__ import annotations
@@ -13,7 +14,7 @@
 from pathlib import Path
 from typing import Any
 
-from metaxy._testing import Runbook
+from metaxy._testing import Runbook, SavedRunbookResult
 
 
 class RunbookLoader:
@@ -260,3 +261,26 @@ def get_patch_snapshots(
         """
         runbook = self.load_runbook(example_name)
         return runbook.get_patch_snapshots()
+
+    def load_execution_result(self, example_name: str) -> SavedRunbookResult:
+        """Load saved execution result from .example.result.json file.
+
+        Args:
+            example_name: Name of the example.
+
+        Returns:
+            Parsed SavedRunbookResult instance.
+
+        Raises:
+            FileNotFoundError: If result file doesn't exist.
+        """
+        example_dir = self.get_example_dir(example_name)
+        result_path = example_dir / ".example.result.json"
+
+        if not result_path.exists():
+            raise FileNotFoundError(
+                f"Execution result not found: {result_path}. "
+                f"Run tests to generate: pytest tests/examples/test_example_snapshots.py"
+            )
+
+        return SavedRunbookResult.from_json_file(result_path)

docs/.mkdocs-metaxy/src/mkdocs_metaxy/examples/markdown_ext.py

@@ -21,6 +21,13 @@
         example: recompute
         path: patches/01_update_parent_algorithm.patch
     :::
+
+    ::: metaxy-example output
+        example: recompute
+        scenario: "Initial pipeline run"
+        step: "run_pipeline"
+        event_type: command
+    :::
 """
 
 from __future__ import annotations
@@ -212,6 +219,8 @@ def _process_directive(self, directive_type: str, content: str) -> str:
             return self._render_file(example_name, params)
         if directive_type == "patch":
             return self._render_patch(example_name, params)
+        if directive_type == "output":
+            return self._render_output(example_name, params)
         raise ValueError(f"Unknown directive type: {directive_type}")
 
     def _render_scenarios(self, example_name: str) -> str:
@@ -334,6 +343,74 @@ def _render_patch(self, example_name: str, params: dict[str, Any]) -> str:
             hl_lines=None,
         )
 
+    def _render_output(self, example_name: str, params: dict[str, Any]) -> str:
+        """Render execution output from saved result.
+
+        Args:
+            example_name: Name of the example.
+            params: Directive parameters (scenario, step, event_type, etc.).
+
+        Returns:
+            Markdown string with execution output.
+
+        Raises:
+            ValueError: If required parameters are missing or invalid.
+        """
+        from metaxy._testing import CommandExecuted, GraphPushed, PatchApplied
+
+        # Load execution result
+        try:
+            result = self.loader.load_execution_result(example_name)
+        except FileNotFoundError as e:
+            return self.renderer.render_error(
+                f"Execution result not found for example '{example_name}'",
+                details=str(e),
+            )
+
+        # Filter events by scenario if specified
+        scenario_name = params.get("scenario")
+        events = result.execution_state.events
+        if scenario_name:
+            events = [e for e in events if e.scenario_name == scenario_name]
+
+        # Filter by step name if specified
+        step_name = params.get("step")
+        if step_name:
+            events = [e for e in events if e.step_name == step_name]
+
+        # Filter by event type if specified
+        event_type = params.get("event_type")
+        if event_type:
+            if event_type == "command":
+                events = [e for e in events if isinstance(e, CommandExecuted)]
+            elif event_type == "patch":
+                events = [e for e in events if isinstance(e, PatchApplied)]
+            elif event_type == "graph_push":
+                events = [e for e in events if isinstance(e, GraphPushed)]
+            else:
+                raise ValueError(f"Unknown event_type: {event_type}")
+
+        # Render all matching events
+        md_parts = []
+        for event in events:
+            if isinstance(event, CommandExecuted):
+                show_command = params.get("show_command", True)
+                md_parts.append(
+                    self.renderer.render_command_output(event, show_command)
+                )
+            elif isinstance(event, PatchApplied):
+                md_parts.append(self.renderer.render_patch_applied(event))
+            elif isinstance(event, GraphPushed):
+                md_parts.append(self.renderer.render_graph_pushed(event))
+
+        if not md_parts:
+            return self.renderer.render_error(
+                "No events matched the specified criteria",
+                details=f"scenario={scenario_name}, step={step_name}, event_type={event_type}",
+            )
+
+        return "\n".join(md_parts)
+
 
 class MetaxyExamplesExtension(Extension):
     """Markdown extension for Metaxy examples."""

docs/.mkdocs-metaxy/src/mkdocs_metaxy/examples/renderer.py

@@ -4,12 +4,15 @@
 - Scenario lists with descriptions
 - Source code in markdown code blocks
 - Diff patches in markdown code blocks
+- Execution events (commands, patches, graph pushes)
 """
 
 from __future__ import annotations
 
 from typing import Any
 
+from metaxy._testing import CommandExecuted, GraphPushed, PatchApplied
+
 
 class ExampleRenderer:
     """Renderer for Metaxy example content."""
@@ -268,3 +271,75 @@ def render_graph_diff(
 
         except Exception:
             return None
+
+    def render_command_output(
+        self, event: CommandExecuted, show_command: bool = True
+    ) -> str:
+        """Render command execution output as markdown.
+
+        Args:
+            event: CommandExecuted event from execution state.
+            show_command: Whether to show the command that was executed.
+
+        Returns:
+            Markdown string with command output.
+        """
+        md_parts = []
+
+        if show_command:
+            md_parts.append(f"```shell\n$ {event.command}\n```")
+            md_parts.append("")
+
+        # Show stdout if present
+        if event.stdout:
+            md_parts.append("```")
+            md_parts.append(event.stdout.rstrip())
+            md_parts.append("```")
+            md_parts.append("")
+
+        # Show stderr if present (as a warning admonition)
+        if event.stderr:
+            md_parts.append('!!! warning "Warnings/Errors"')
+            md_parts.append("    ```")
+            for line in event.stderr.rstrip().split("\n"):
+                md_parts.append(f"    {line}")
+            md_parts.append("    ```")
+            md_parts.append("")
+
+        return "\n".join(md_parts)
+
+    def render_patch_applied(self, event: PatchApplied) -> str:
+        """Render patch application event as markdown.
+
+        Args:
+            event: PatchApplied event from execution state.
+
+        Returns:
+            Markdown string describing the patch application.
+        """
+        md_parts = []
+
+        md_parts.append(f"**Applied patch:** `{event.patch_path}`")
+        md_parts.append("")
+
+        if event.before_snapshot and event.after_snapshot:
+            before_short = event.before_snapshot[:8]
+            after_short = event.after_snapshot[:8]
+            md_parts.append(
+                f"Graph snapshot changed: `{before_short}...` → `{after_short}...`"
+            )
+            md_parts.append("")
+
+        return "\n".join(md_parts)
+
+    def render_graph_pushed(self, event: GraphPushed) -> str:
+        """Render graph push event as markdown.
+
+        Args:
+            event: GraphPushed event from execution state.
+
+        Returns:
+            Markdown string describing the graph push.
+        """
+        snapshot_short = event.snapshot_version[:8]
+        return f"**Graph snapshot recorded:** `{snapshot_short}...`\n\n"

examples/example-one-to-many/.example.result.json

@@ -0,0 +1,58 @@
+{
+  "runbook_name": "One-to-Many Expansion Example",
+  "package_name": "example_one_to_many",
+  "description": "Demonstrates 1:N expansion relationships and field-level provenance tracking",
+  "execution_state": {
+    "events": [
+      {
+        "type": "command_executed",
+        "command": "python pipeline.py",
+        "returncode": 0,
+        "stdout": "Found 3 new videos\nFound 3 videos and 0 videos that need chunking\nProcessing video: {'video_id': 1, 'path': 'video1.mp4', 'metaxy_provenance_by_field__video_raw': {'audio': 'v1', 'frames': 'v1'}, 'metaxy_provenance__video_raw': 'c0e1a00f0a117623634caf2ae3bca98d', 'metaxy_provenance_by_field': {'audio': '4d4f04548ded52073a3e1acdc174119b', 'frames': '7543ba636a8491695e298d97b6f5d76e'}, 'metaxy_provenance': 'b70689cc645b36a1043cfac8ac279339'}\nWriting 5 chunks for video 1\nProcessing video: {'video_id': 2, 'path': 'video2.mp4', 'metaxy_provenance_by_field__video_raw': {'audio': 'v2', 'frames': 'v2'}, 'metaxy_provenance__video_raw': '6086bb90a046bc4b8a66aae64458f905', 'metaxy_provenance_by_field': {'audio': 'a79c4b1dd347aa59a478b59db84fb501', 'frames': '141f827b4c92e423b34cdec12a517c1b'}, 'metaxy_provenance': '727b940f16e01d887661376f5b172591'}\nWriting 3 chunks for video 2\nProcessing video: {'video_id': 3, 'path': 'video3.mp4', 'metaxy_provenance_by_field__video_raw': {'audio': 'v3', 'frames': 'v3'}, 'metaxy_provenance__video_raw': 'ba24e4a269e9b58c00414bc3dd8ca9f5', 'metaxy_provenance_by_field': {'audio': 'd7bb40cd544c105f2e2fbaccf6538b8d', 'frames': 'b09c343e918fff13153559bb0a09f82d'}, 'metaxy_provenance': '256503a3e5370c25696f88e2ac802d46'}\nWriting 3 chunks for video 3\nFound 11 video chunks and 0 video chunks that need face recognition\nWriting face recognition results for 11 chunks\n",
+        "stderr": "/home/dan/code/github/anam-org/metaxy/examples/example-one-to-many/pipeline.py:116: UserWarning: AUTO_CREATE_TABLES is enabled for IbisMetadataStore(backend=duckdb) - do not use in production! Use proper database migration tools like Alembic for production deployments.\n  main()\nFeature video/raw: samples parameter is Polars-backed but store uses native SQL backend. Materializing current metadata to Polars for diff comparison. For better performance, consider using samples with backend matching the store's backend.\n/home/dan/code/github/anam-org/metaxy/examples/example-one-to-many/pipeline.py:116: UserWarning: AUTO_CREATE_TABLES is enabled for IbisMetadataStore(backend=duckdb) - do not use in production! Use proper database migration tools like Alembic for production deployments.\n  main()\n",
+        "timestamp": "2025-11-11T20:01:21.702103",
+        "scenario_name": "Initial pipeline run",
+        "step_name": "initial_run"
+      },
+      {
+        "type": "command_executed",
+        "command": "python pipeline.py",
+        "returncode": 0,
+        "stdout": "Found 0 videos and 0 videos that need chunking\nFound 0 video chunks and 0 video chunks that need face recognition\n",
+        "stderr": "/home/dan/code/github/anam-org/metaxy/examples/example-one-to-many/pipeline.py:116: UserWarning: AUTO_CREATE_TABLES is enabled for IbisMetadataStore(backend=duckdb) - do not use in production! Use proper database migration tools like Alembic for production deployments.\n  main()\nFeature video/raw: samples parameter is Polars-backed but store uses native SQL backend. Materializing current metadata to Polars for diff comparison. For better performance, consider using samples with backend matching the store's backend.\n/home/dan/code/github/anam-org/metaxy/examples/example-one-to-many/pipeline.py:116: UserWarning: AUTO_CREATE_TABLES is enabled for IbisMetadataStore(backend=duckdb) - do not use in production! Use proper database migration tools like Alembic for production deployments.\n  main()\n",
+        "timestamp": "2025-11-11T20:01:23.193091",
+        "scenario_name": "Idempotent rerun",
+        "step_name": "idempotent_run"
+      },
+      {
+        "type": "patch_applied",
+        "patch_path": "patches/01_update_video_code_version.patch",
+        "before_snapshot": null,
+        "after_snapshot": null,
+        "timestamp": "2025-11-11T20:01:24.386038",
+        "scenario_name": "Code change - audio field only",
+        "step_name": "update_audio_version"
+      },
+      {
+        "type": "command_executed",
+        "command": "python pipeline.py",
+        "returncode": 0,
+        "stdout": "Found 3 new videos\nFound 3 videos and 0 videos that need chunking\nProcessing video: {'video_id': 1, 'path': 'video1.mp4', 'metaxy_provenance_by_field__video_raw': {'audio': 'v1', 'frames': 'v1'}, 'metaxy_provenance__video_raw': 'c0e1a00f0a117623634caf2ae3bca98d', 'metaxy_provenance_by_field': {'audio': '4d4f04548ded52073a3e1acdc174119b', 'frames': '7543ba636a8491695e298d97b6f5d76e'}, 'metaxy_provenance': 'b70689cc645b36a1043cfac8ac279339'}\nWriting 5 chunks for video 1\nProcessing video: {'video_id': 2, 'path': 'video2.mp4', 'metaxy_provenance_by_field__video_raw': {'audio': 'v2', 'frames': 'v2'}, 'metaxy_provenance__video_raw': '6086bb90a046bc4b8a66aae64458f905', 'metaxy_provenance_by_field': {'audio': 'a79c4b1dd347aa59a478b59db84fb501', 'frames': '141f827b4c92e423b34cdec12a517c1b'}, 'metaxy_provenance': '727b940f16e01d887661376f5b172591'}\nWriting 3 chunks for video 2\nProcessing video: {'video_id': 3, 'path': 'video3.mp4', 'metaxy_provenance_by_field__video_raw': {'audio': 'v3', 'frames': 'v3'}, 'metaxy_provenance__video_raw': 'ba24e4a269e9b58c00414bc3dd8ca9f5', 'metaxy_provenance_by_field': {'audio': 'd7bb40cd544c105f2e2fbaccf6538b8d', 'frames': 'b09c343e918fff13153559bb0a09f82d'}, 'metaxy_provenance': '256503a3e5370c25696f88e2ac802d46'}\nWriting 3 chunks for video 3\nFound 0 video chunks and 0 video chunks that need face recognition\n",
+        "stderr": "/home/dan/code/github/anam-org/metaxy/examples/example-one-to-many/pipeline.py:116: UserWarning: AUTO_CREATE_TABLES is enabled for IbisMetadataStore(backend=duckdb) - do not use in production! Use proper database migration tools like Alembic for production deployments.\n  main()\nFeature video/raw: samples parameter is Polars-backed but store uses native SQL backend. Materializing current metadata to Polars for diff comparison. For better performance, consider using samples with backend matching the store's backend.\n/home/dan/code/github/anam-org/metaxy/examples/example-one-to-many/pipeline.py:116: UserWarning: AUTO_CREATE_TABLES is enabled for IbisMetadataStore(backend=duckdb) - do not use in production! Use proper database migration tools like Alembic for production deployments.\n  main()\n",
+        "timestamp": "2025-11-11T20:01:25.969737",
+        "scenario_name": "Code change - audio field only",
+        "step_name": "recompute_after_audio_change"
+      },
+      {
+        "type": "command_executed",
+        "command": "python pipeline.py",
+        "returncode": 0,
+        "stdout": "Found 0 videos and 0 videos that need chunking\nFound 0 video chunks and 0 video chunks that need face recognition\n",
+        "stderr": "/home/dan/code/github/anam-org/metaxy/examples/example-one-to-many/pipeline.py:116: UserWarning: AUTO_CREATE_TABLES is enabled for IbisMetadataStore(backend=duckdb) - do not use in production! Use proper database migration tools like Alembic for production deployments.\n  main()\nFeature video/raw: samples parameter is Polars-backed but store uses native SQL backend. Materializing current metadata to Polars for diff comparison. For better performance, consider using samples with backend matching the store's backend.\n/home/dan/code/github/anam-org/metaxy/examples/example-one-to-many/pipeline.py:116: UserWarning: AUTO_CREATE_TABLES is enabled for IbisMetadataStore(backend=duckdb) - do not use in production! Use proper database migration tools like Alembic for production deployments.\n  main()\n",
+        "timestamp": "2025-11-11T20:01:27.463343",
+        "scenario_name": "Final idempotent check",
+        "step_name": "final_idempotent_check"
+      }
+    ]
+  }
+}

examples/example-one-to-many/.example.yaml

@@ -6,7 +6,8 @@ scenarios:
   - name: "Initial pipeline run"
     description: "First execution creates videos, chunks, and face recognition"
     steps:
-      - type: "run_command"
+      - name: "initial_run"
+        type: "run_command"
         command: "python pipeline.py"
         description: "Run pipeline to create videos, chunks, and face recognition"
         capture_output: true
@@ -26,7 +27,8 @@ scenarios:
   - name: "Idempotent rerun"
     description: "Second execution should detect no changes"
     steps:
-      - type: "run_command"
+      - name: "idempotent_run"
+        type: "run_command"
         command: "python pipeline.py"
         description: "Rerun pipeline without any code changes"
         capture_output: true
@@ -42,11 +44,13 @@ scenarios:
   - name: "Code change - audio field only"
     description: "Apply patch to change Video audio code_version, observe field-level tracking"
     steps:
-      - type: "apply_patch"
+      - name: "update_audio_version"
+        type: "apply_patch"
         patch_path: "patches/01_update_video_code_version.patch"
         description: "Change Video audio field code_version from 1 to 2"
 
-      - type: "run_command"
+      - name: "recompute_after_audio_change"
+        type: "run_command"
         command: "python pipeline.py"
         description: "Run pipeline after audio code_version change"
         capture_output: true
@@ -64,7 +68,8 @@ scenarios:
   - name: "Final idempotent check"
     description: "Verify no further recomputation needed after code change processed"
     steps:
-      - type: "run_command"
+      - name: "final_idempotent_check"
+        type: "run_command"
         command: "python pipeline.py"
         description: "Rerun pipeline after code change"
         capture_output: true

examples/example-one-to-many/pipeline.py

@@ -1,3 +1,4 @@
+import os
 import random
 
 import narwhals as nw
@@ -9,6 +10,9 @@
 
 
 def main():
+    # Set random seed from environment if provided (for deterministic testing)
+    if seed_str := os.environ.get("RANDOM_SEED"):
+        random.seed(int(seed_str))
     cfg = init_metaxy()
     store = cfg.get_store("dev")