CI: only run playbook tests whose behavior actually changed

.github/scripts/build_test_matrix.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+# Copyright Advanced Micro Devices, Inc.
+#
+# SPDX-License-Identifier: MIT
+
+"""
+Test Matrix Builder
+===================
+
+Builds the CI test matrix for test-playbooks.yml.
+
+Modes:
+    --mode all                              every entry (nightly cron)
+    --mode playbook --playbook-id <id>      one playbook (workflow_dispatch)
+    --mode changed --base <sha>             only entries whose tests changed (pull_request)
+
+How --mode changed works:
+    1. Materialise the base revision (git archive) into a temp tree.
+    2. Harness check: if any file that governs how tests execute differs between
+       base and head, run the FULL matrix. This is deny-by-default over
+       .github/scripts/ and test-playbooks.yml, minus HARNESS_EXCLUDED.
+    3. Otherwise the extractor is identical on both sides, so compute a per-entry
+       signature with THIS checkout's extractor against each tree and diff them.
+       An entry runs if it is new, its matrix identity changed, its signature
+       changed, or either signature could not be computed.
+
+    Applying one extractor to both trees is only safe because step 2 already
+    forced the full matrix on any extractor change; that is what prevents a
+    semantic change from cancelling out. Every failure to establish a trustworthy
+    base (missing sha, archive error) falls back to the full matrix.
+"""
+
+import argparse
+import hashlib
+import io
+import json
+import os
+import subprocess
+import sys
+import tarfile
+import tempfile
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+import run_playbook_tests as R  # noqa: E402
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+ARCHIVE_TIMEOUT = 120
+
+# Files that govern how tests execute. Deny-by-default: anything added under
+# these prefixes is covered without anyone remembering. Editing any of them
+# forces the full matrix, so signatures never have to model harness behaviour.
+HARNESS_PREFIXES = (".github/scripts/", ".github/workflows/test-playbooks.yml")
+
+# Files under those prefixes that provably cannot change a playbook test verdict.
+# Omitting one only costs a needless full run; wrongly adding one hides a real
+# change, so each entry is justified by test-playbooks.yml not depending on it.
+HARNESS_EXCLUDED = frozenset({
+    ".github/scripts/translate_playbook.py",       # translation tooling
+    ".github/scripts/disclaimers.json",            # translation tooling
+    ".github/scripts/glossary.json",               # translation tooling
+    ".github/scripts/check_copyright.py",          # check-copyright.yml
+    ".github/scripts/validate_playbooks.py",       # validate-playbooks.yml
+    ".github/scripts/fetch_github_issues.py",      # fetch-github-issues.yml
+    ".github/scripts/select_runners.py",           # not referenced by test-playbooks.yml
+    ".github/scripts/create_failure_issues.py",    # if: failure() on main, post-verdict
+    ".github/scripts/test_build_test_matrix.py",   # separate selector-tests job
+    ".github/scripts/gen_issue_template_playbooks.py",  # validate-playbooks.yml
+    ".github/scripts/orchestrai_matrix.py",        # OrchestrAI workflows
+    ".github/scripts/orchestrai_report.py",        # OrchestrAI workflows
+    ".github/scripts/orchestrai_trigger.py",       # OrchestrAI workflows
+    ".github/scripts/orchestrai_verdict.py",       # OrchestrAI workflows
+})
+
+def annotate(level: str, message: str) -> None:
+    """Report on stderr only; stdout is reserved for the matrix JSON."""
+    prefix = f"::{level}::" if os.environ.get("GITHUB_ACTIONS") else ""
+    print(f"{prefix}{message}", file=sys.stderr)
+
+
+def harness_files(root: Path) -> dict[str, bytes]:
+    """Map relpath -> content hash for every execution-governing file in root."""
+    out: dict[str, bytes] = {}
+    for prefix in HARNESS_PREFIXES:
+        target = root / prefix
+        paths = []
+        if target.is_dir():
+            paths = [p for p in target.rglob("*") if p.is_file()]
+        elif target.is_file():
+            paths = [target]
+        for path in paths:
+            rel = str(path.relative_to(root)).replace(os.sep, "/")
+            if "__pycache__" in path.relative_to(root).parts or rel.endswith(".pyc"):
+                continue
+            if rel in HARNESS_EXCLUDED:
+                continue
+            mode = b"\x01" if os.access(path, os.X_OK) else b"\x00"
+            out[rel] = mode + hashlib.sha256(path.read_bytes()).digest()
+    return out
+
+
+def harness_changed(base_tree: Path) -> bool:
+    return harness_files(REPO_ROOT) != harness_files(base_tree)
+
+
+def materialise_base(base: str, dest: Path) -> bool:
+    """Extract the base revision's playbooks and .github into dest."""
+    try:
+        archive = subprocess.run(
+            ["git", "archive", "--format=tar", base, "--", "playbooks", ".github"],
+            cwd=REPO_ROOT, capture_output=True, check=True,
+            stdin=subprocess.DEVNULL, timeout=ARCHIVE_TIMEOUT,
+        )
+    except subprocess.TimeoutExpired:
+        annotate("warning", f"git archive {base} timed out after {ARCHIVE_TIMEOUT}s")
+        return False
+    except subprocess.CalledProcessError as exc:
+        annotate("warning", f"git archive {base} failed: {exc.stderr.decode()[:200]}")
+        return False
+    try:
+        with tarfile.open(fileobj=io.BytesIO(archive.stdout)) as tar:
+            tar.extractall(dest, filter="data")
+    except Exception as exc:
+        annotate("warning", f"cannot extract base {base}: {type(exc).__name__}: {exc}")
+        return False
+    return True
+
+
+def select_changed(base_tree: Path) -> list[dict]:
+    """Entries not provably unchanged between base_tree and this checkout."""
+    head = R.build_matrix_entries(REPO_ROOT)
+    if harness_changed(base_tree):
+        annotate("notice", "Test harness changed; running the full matrix")
+        return head
+
+    base_keys = {
+        (e["playbook"], e["platform"], e["arch"]): e
+        for e in R.build_matrix_entries(base_tree)
+    }
+    selected, skipped = [], 0
+    for entry in head:
+        key = (entry["playbook"], entry["platform"], entry["arch"])
+        label = f"{key[0]} ({key[1]}/{key[2]})"
+        prior = base_keys.get(key)
+        if prior is None:
+            annotate("notice", f"New matrix entry {label}")
+        elif prior != entry:
+            annotate("notice", f"Matrix metadata changed for {label}")
+        else:
+            head_sig = R.entry_signature(REPO_ROOT, *key)
+            base_sig = R.entry_signature(base_tree, *key)
+            if head_sig is None or base_sig is None:
+                annotate("warning", f"Cannot prove {label} unchanged; running it")
+            elif head_sig != base_sig:
+                annotate("notice", f"Tests changed for {label}")
+            else:
+                skipped += 1
+                continue
+        selected.append(entry)
+
+    # An entry that vanished while its playbook still exists is still runnable
+    head_keys = {(e["playbook"], e["platform"], e["arch"]) for e in head}
+    live = set(R.list_playbook_ids(REPO_ROOT))
+    for key, prior in base_keys.items():
+        if key not in head_keys and key[0] in live:
+            annotate("warning", f"Entry vanished from {key[0]} ({key[1]}/{key[2]}) "
+                                "but the playbook still exists; running it")
+            selected.append(prior)
+
+    annotate("notice", f"Selected {len(selected)} entries; skipped {skipped} unchanged")
+    return selected
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Build the playbook CI test matrix")
+    parser.add_argument("--mode", required=True, choices=["all", "playbook", "changed"])
+    parser.add_argument("--playbook-id", help="Playbook id for --mode playbook")
+    parser.add_argument("--base", help="Base revision for --mode changed")
+    parser.add_argument("--github-output", type=Path)
+    args = parser.parse_args()
+
+    entries = R.build_matrix_entries(REPO_ROOT)
+    if not entries:
+        print("FATAL: this checkout lists no matrix entries", file=sys.stderr)
+        sys.exit(1)
+
+    if args.mode == "playbook":
+        if not args.playbook_id:
+            parser.error("--mode playbook requires --playbook-id")
+        entries = [e for e in entries if e["playbook"] == args.playbook_id]
+        if not entries:
+            print(f"FATAL: no matrix entries for playbook '{args.playbook_id}'", file=sys.stderr)
+            sys.exit(1)
+    elif args.mode == "changed":
+        if not args.base:
+            parser.error("--mode changed requires --base")
+        with tempfile.TemporaryDirectory(prefix="base-tree-") as tmp:
+            base_tree = Path(tmp)
+            if materialise_base(args.base, base_tree):
+                entries = select_changed(base_tree)
+            else:
+                annotate("warning", f"No trustworthy base {args.base}; running the full matrix")
+
+    matrix = json.dumps(entries)
+    output_path = args.github_output or (
+        Path(os.environ["GITHUB_OUTPUT"]) if os.environ.get("GITHUB_OUTPUT") else None
+    )
+    if output_path:
+        with output_path.open("a", encoding="utf-8") as handle:
+            handle.write(f"matrix={matrix}\n")
+            handle.write(f"has_entries={'true' if entries else 'false'}\n")
+            handle.write("detection_ok=true\n")
+    print(matrix)
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/run_playbook_tests.py

@@ -150,6 +150,10 @@
 """
 
 import argparse
+import contextlib
+import dataclasses
+import hashlib
+import io
 import json
 import os
 import re
@@ -160,6 +164,9 @@
 from pathlib import Path
 from typing import Optional
 
+# TestBlock fields that do not affect execution, so they stay out of signatures
+SIGNATURE_EXCLUDED = frozenset({"line_number", "hidden"})
+
 
 VALID_DEVICES = {"halo", "stx", "krk", "rx7900xt", "rx9070xt", "r9700"}
 
@@ -204,9 +211,10 @@ class PlaybookTestSuite:
     results: list[TestResult] = field(default_factory=list)
 
 
-def find_playbook_path(playbook_id: str) -> Optional[Path]:
+def find_playbook_path(playbook_id: str, repo_root: Optional[Path] = None) -> Optional[Path]:
     """Find the playbook directory by ID."""
-    repo_root = Path(__file__).parent.parent.parent
+    # repo_root is injectable so the selector can read a base checkout too
+    repo_root = repo_root or Path(__file__).parent.parent.parent
 
     # Check core and supplemental directories
     for category in ["core", "supplemental"]:
@@ -472,15 +480,15 @@ def _replace(match: re.Match) -> str:
     return placeholder_pattern.sub(_replace, code)
 
 
-def resolve_require_tags(content: str) -> str:
+def resolve_require_tags(content: str, repo_root: Optional[Path] = None) -> str:
     """Resolve @require tags by inlining dependency content.
 
     Finds ``<!-- @require:dep-id -->`` tags in the README content and replaces
     them with the actual dependency file contents from the central
     ``playbooks/dependencies/`` folder.  This allows the test extractor to
     discover @test blocks that live inside shared dependency files.
     """
-    repo_root = Path(__file__).parent.parent.parent
+    repo_root = repo_root or Path(__file__).parent.parent.parent
     dependencies_root = repo_root / "playbooks" / "dependencies"
     registry_path = dependencies_root / "registry.json"
 
@@ -591,12 +599,13 @@ def _infer_device(content: str, position: int) -> str:
     return "all"
 
 
-def extract_tests(readme_path: Path, target_platform: str, target_device: Optional[str] = None) -> list[TestBlock]:
+def extract_tests(readme_path: Path, target_platform: str, target_device: Optional[str] = None,
+                  repo_root: Optional[Path] = None) -> list[TestBlock]:
     """Extract test blocks from a README.md file."""
     content = readme_path.read_text(encoding="utf-8")
 
     # Resolve @require tags so tests inside dependencies are discovered
-    content = resolve_require_tags(content)
+    content = resolve_require_tags(content, repo_root)
 
     tests = []
 
@@ -1073,6 +1082,107 @@ def run_playbook_tests(playbook_id: str, platform: str, device: Optional[str] =
     return all_passed
 
 
+# --- Test selection support -------------------------------------------------
+# The matrix builder (build_test_matrix.py) imports these to compute, for one
+# checkout, what each (playbook, platform, device) entry would execute. It runs
+# them against both the head tree and a materialised base tree using THIS
+# extractor, so only content differs between the two sides. A separate harness
+# path check forces the full matrix whenever the extractor itself changed, which
+# is what makes applying one extractor to both trees safe.
+
+def list_playbook_ids(repo_root: Path) -> list[str]:
+    ids = set()
+    for category in ("core", "supplemental"):
+        base = repo_root / "playbooks" / category
+        if base.is_dir():
+            ids.update(p.name for p in base.iterdir() if (p / "playbook.json").exists())
+    return sorted(ids)
+
+
+def build_matrix_entries(repo_root: Path) -> list[dict]:
+    """Expand every playbook.json into CI matrix entries."""
+    entries = []
+    for playbook_id in list_playbook_ids(repo_root):
+        for category in ("core", "supplemental"):
+            meta_file = repo_root / "playbooks" / category / playbook_id / "playbook.json"
+            if not meta_file.exists():
+                continue
+            try:
+                meta = json.loads(meta_file.read_text(encoding="utf-8"))
+            except (OSError, json.JSONDecodeError) as exc:
+                print(f"Warning: cannot read {meta_file}: {exc}", file=sys.stderr)
+                break
+            required = meta.get("required_platforms", {})
+            for device, platforms in meta.get("tested_platforms", {}).items():
+                for platform in platforms:
+                    os_label = "Windows" if platform == "windows" else "Linux"
+                    entries.append({
+                        "playbook": playbook_id,
+                        "platform": platform,
+                        "arch": device,
+                        "runner": json.dumps(["self-hosted", os_label, device]),
+                        "required": platform in set(required.get(device, [])),
+                    })
+            break
+    return entries
+
+
+def assets_digest(assets_dir: Path) -> Optional[str]:
+    """Digest an assets tree. None means "cannot be signed", which forces a run.
+
+    A symlink escaping the tree is unsignable: its target content is not covered
+    here, so a change to that target would otherwise be invisible.
+    """
+    if not assets_dir.is_dir():
+        return ""
+    root = assets_dir.resolve()
+    digest = hashlib.sha256()
+    for path in sorted(assets_dir.rglob("*")):
+        rel = str(path.relative_to(assets_dir)).replace(os.sep, "/")
+        if path.is_symlink():
+            try:
+                target = path.resolve()
+            except OSError:
+                return None
+            if not target.is_relative_to(root):
+                return None
+            digest.update(rel.encode("utf-8") + b"\x02" + os.readlink(path).encode("utf-8"))
+        elif path.is_file():
+            digest.update(rel.encode("utf-8"))
+            digest.update(b"\x01" if os.access(path, os.X_OK) else b"\x00")
+            digest.update(hashlib.sha256(path.read_bytes()).digest())
+    return digest.hexdigest()
+
+
+def entry_signature(repo_root: Path, playbook_id: str, platform: str,
+                    device: str) -> Optional[str]:
+    """Signature of what a checkout would execute for one entry, or None.
+
+    None means the signature could not be computed (missing playbook, extraction
+    error); the caller must then treat the entry as not-provably-equal and run it.
+    """
+    playbook_path = find_playbook_path(playbook_id, repo_root=repo_root)
+    if playbook_path is None:
+        return None
+    try:
+        with contextlib.redirect_stdout(io.StringIO()):
+            tests = extract_tests(playbook_path / "README.md", platform, device,
+                                  repo_root=repo_root)
+    except Exception:
+        return None
+    # @require inlines shared dependency markdown, so its assets are in scope too
+    own = assets_digest(playbook_path / "assets")
+    shared = assets_digest(repo_root / "playbooks" / "dependencies" / "assets")
+    if own is None or shared is None:
+        return None
+    payload = [
+        {k: v for k, v in dataclasses.asdict(t).items() if k not in SIGNATURE_EXCLUDED}
+        for t in tests
+    ]
+    blob = json.dumps([payload, own, shared], sort_keys=True)
+    return hashlib.sha256(blob.encode("utf-8")).hexdigest()
+
+
 def main():
     parser = argparse.ArgumentParser(description="Run playbook tests")
     parser.add_argument("--playbook", required=True, help="Playbook ID to test")