Gate search compatibility on result overlap

tests/search-backends/README.md

@@ -22,10 +22,12 @@ private-to-public publication, removal, and an observable index completion.
 
 ## Automation
 
-The `Integration testing` workflow runs the focused indexed-HTML contract on
+The `Integration testing` workflow runs the focused indexed-search contract on
 every push for Virtuoso with SBOLExplorer, sbol-db with SBOLExplorer, and
 sbol-db with its compatibility listener. Each row explicitly rebuilds the
-configured index and compares the complete `/search/I0462` HTML snapshot.
+configured index and requires at least 50% Jaccard overlap between stable
+result identities from `/search/I0462`; page markup and result metadata are
+not compared.
 
 The `Search backend conformance` workflow runs the full pinned corpus every
 Sunday and on manual dispatch. It runs the SBOLExplorer baseline and sbol-db
@@ -91,9 +93,10 @@ python3 tests/search-backends/compare-reports.py \
 
 The comparison records submission and search-result differences and applies a
 pinned-corpus compatibility policy. Both lifecycle gates and identical
-submission outcomes are required; at least 90% of complete first-page result
-sets must agree, no more than two probes may have count drift, and either drift
-may be at most one result. Exact top-ten order remains diagnostic because
-SBOLExplorer and sbol-db intentionally use different ranking implementations.
-This boundary catches tokenizer explosions and missing ontology enrichment
-without pretending the two rankers are byte-identical.
+submission outcomes are required. At least 90% of probes must have 50% or
+greater Jaccard overlap between first-page result identities. Exact result
+sets, counts, metadata, and top-ten order remain diagnostic because
+SBOLExplorer and sbol-db intentionally use different search and ranking
+implementations. Jaccard overlap still penalizes unrelated extra results, so a
+first page dominated by unrelated objects will not satisfy the policy; full
+result counts remain visible in the diagnostic report.

tests/search-backends/compare-reports.py

@@ -9,9 +9,8 @@
 import sys
 
 
-MIN_EXACT_FIRST_PAGE_RATIO = 0.90
-MAX_COUNT_DIFFERENCE_PROBES = 2
-MAX_ABSOLUTE_COUNT_DELTA = 1
+MIN_FIRST_PAGE_IDENTITY_OVERLAP = 0.50
+MIN_SUBSTANTIAL_OVERLAP_PROBE_RATIO = 0.90
 
 
 def status_class(status: int | None) -> int | None:
@@ -32,6 +31,22 @@ def signature_sort_key(signature: tuple[object, ...]) -> tuple[str, ...]:
     return tuple("" if value is None else str(value) for value in signature)
 
 
+def result_identities(results: list[dict[str, object]]) -> set[str]:
+    """Return stable result identities without comparing rendered metadata."""
+    return {
+        str(result["uri_path"])
+        for result in results
+        if result.get("uri_path") is not None
+    }
+
+
+def jaccard_overlap(left: set[str], right: set[str]) -> float:
+    """Measure shared identities while penalizing unrelated extra results."""
+    if not left and not right:
+        return 1.0
+    return len(left & right) / len(left | right)
+
+
 def keyed(report: dict[str, object], field: str) -> dict[str, dict[str, object]]:
     return {str(entry["path"]): entry for entry in report[field]}
 
@@ -41,50 +56,50 @@ def evaluate_parity(
     baseline_gate_passed: bool,
     candidate_gate_passed: bool,
     summary: dict[str, int | float],
-    probe_differences: list[dict[str, object]],
 ) -> dict[str, object]:
-    """Apply a pinned-corpus compatibility policy without requiring ES order.
+    """Apply a pinned-corpus compatibility policy across distinct rankers.
 
     sbol-db and SBOLExplorer use different ranking implementations, so exact
-    top-ten order is diagnostic evidence rather than a compatibility gate. The
-    policy does require almost all complete first pages to agree, tightly
-    bounds count drift, and independently requires both lifecycle suites.
+    result sets, counts, metadata, and top-ten order are diagnostic evidence.
+    The compatibility gate instead requires substantial first-page identity
+    overlap for almost all probes and independently requires both lifecycle
+    suites.
     """
     compared = int(summary["compared_probe_count"])
     exact = int(summary["exact_first_page_parity_count"])
     exact_ratio = exact / compared if compared else 0.0
-    count_deltas = [
-        abs(int(difference["baseline_count"]) - int(difference["candidate_count"]))
-        for difference in probe_differences
-        if difference.get("baseline_count") is not None
-        and difference.get("candidate_count") is not None
-        and difference["baseline_count"] != difference["candidate_count"]
-    ]
-    max_count_delta = max(count_deltas, default=0)
+    substantial = int(summary["substantial_first_page_overlap_count"])
+    substantial_ratio = substantial / compared if compared else 0.0
     checks = {
         "same_pinned_corpus": corpus_equal,
         "baseline_conformance_passed": baseline_gate_passed,
         "candidate_conformance_passed": candidate_gate_passed,
         "submission_outcomes_identical": summary["submission_difference_count"] == 0,
-        "exact_first_page_ratio_at_least_90_percent": (
-            exact_ratio >= MIN_EXACT_FIRST_PAGE_RATIO
-        ),
-        "count_difference_probes_at_most_2": (
-            summary["count_difference_count"] <= MAX_COUNT_DIFFERENCE_PROBES
+        "substantial_first_page_overlap_ratio_at_least_90_percent": (
+            substantial_ratio >= MIN_SUBSTANTIAL_OVERLAP_PROBE_RATIO
         ),
-        "maximum_count_delta_at_most_1": max_count_delta <= MAX_ABSOLUTE_COUNT_DELTA,
     }
     return {
         "checks": checks,
         "passed": all(checks.values()),
         "observed": {
             "exact_first_page_ratio": exact_ratio,
-            "maximum_count_delta": max_count_delta,
+            "substantial_first_page_overlap_ratio": substantial_ratio,
+            "mean_first_page_identity_overlap": summary[
+                "mean_first_page_identity_overlap"
+            ],
+            "minimum_first_page_identity_overlap": summary[
+                "minimum_first_page_identity_overlap"
+            ],
+            "maximum_count_delta": summary["maximum_count_delta"],
         },
         "policy": {
-            "minimum_exact_first_page_ratio": MIN_EXACT_FIRST_PAGE_RATIO,
-            "maximum_count_difference_probes": MAX_COUNT_DIFFERENCE_PROBES,
-            "maximum_absolute_count_delta": MAX_ABSOLUTE_COUNT_DELTA,
+            "minimum_first_page_identity_overlap": MIN_FIRST_PAGE_IDENTITY_OVERLAP,
+            "minimum_substantial_overlap_probe_ratio": (
+                MIN_SUBSTANTIAL_OVERLAP_PROBE_RATIO
+            ),
+            "exact_result_sets_are_diagnostic_only": True,
+            "result_counts_are_diagnostic_only": True,
             "top_10_order_is_diagnostic_only": True,
         },
     }
@@ -128,11 +143,19 @@ def main() -> int:
     candidate_probes = keyed(candidate, "probes")
     compared_probe_paths = set(baseline_probes) | set(candidate_probes)
     probe_differences = []
+    first_page_overlaps = []
+    count_deltas = []
     for path in sorted(compared_probe_paths):
         left = baseline_probes.get(path, {})
         right = candidate_probes.get(path, {})
         left_set = {result_signature(row) for row in left.get("results", [])}
         right_set = {result_signature(row) for row in right.get("results", [])}
+        left_identities = result_identities(left.get("results", []))
+        right_identities = result_identities(right.get("results", []))
+        identity_overlap = jaccard_overlap(left_identities, right_identities)
+        first_page_overlaps.append(identity_overlap)
+        if left.get("count") is not None and right.get("count") is not None:
+            count_deltas.append(abs(int(left["count"]) - int(right["count"])))
         if left.get("count") != right.get("count") or left_set != right_set:
             probe_differences.append(
                 {
@@ -151,6 +174,7 @@ def main() -> int:
                             right_set - left_set, key=signature_sort_key
                         )
                     ],
+                    "first_page_identity_overlap": identity_overlap,
                     "top_10_order_equal": [
                         result_signature(row) for row in left.get("results", [])[:10]
                     ]
@@ -163,11 +187,22 @@ def main() -> int:
         "compared_probe_count": len(compared_probe_paths),
         "exact_first_page_parity_count": len(compared_probe_paths)
         - len(probe_differences),
+        "substantial_first_page_overlap_count": sum(
+            overlap >= MIN_FIRST_PAGE_IDENTITY_OVERLAP
+            for overlap in first_page_overlaps
+        ),
+        "mean_first_page_identity_overlap": (
+            sum(first_page_overlaps) / len(first_page_overlaps)
+            if first_page_overlaps
+            else 0.0
+        ),
+        "minimum_first_page_identity_overlap": min(first_page_overlaps, default=0.0),
         "probe_difference_count": len(probe_differences),
         "count_difference_count": sum(
             difference["baseline_count"] != difference["candidate_count"]
             for difference in probe_differences
         ),
+        "maximum_count_delta": max(count_deltas, default=0),
         "top_10_order_difference_count": sum(
             difference["top_10_order_equal"] is False
             for difference in probe_differences
@@ -178,10 +213,9 @@ def main() -> int:
         baseline["gate"]["passed"],
         candidate["gate"]["passed"],
         summary,
-        probe_differences,
     )
     output = {
-        "schema_version": 3,
+        "schema_version": 4,
         "baseline": baseline["topology"],
         "candidate": candidate["topology"],
         "corpus_equal": corpus_equal,
@@ -193,9 +227,10 @@ def main() -> int:
         "parity_gate": parity_gate,
         "passed": parity_gate["passed"],
         "note": (
-            "Exact top-ten order remains diagnostic because the implementations "
-            "use different rankers; lifecycle, result coverage, first-page set "
-            "agreement, and tightly bounded count drift are required."
+            "Exact result sets, counts, metadata, and top-ten order remain "
+            "diagnostic because the implementations use different rankers; "
+            "lifecycle, result coverage, and substantial first-page identity "
+            "overlap are required."
         ),
     }
     args.output.parent.mkdir(parents=True, exist_ok=True)

tests/search-backends/test_compare_reports.py

@@ -1,5 +1,9 @@
 import importlib.util
+import json
 from pathlib import Path
+import subprocess
+import sys
+import tempfile
 import unittest
 
 
@@ -10,36 +14,133 @@
 
 
 class ParityPolicyTest(unittest.TestCase):
-    def test_accepts_current_compatibility_boundary(self):
+    def test_accepts_substantial_overlap_despite_count_drift(self):
         summary = {
             "compared_probe_count": 284,
             "exact_first_page_parity_count": 261,
+            "substantial_first_page_overlap_count": 263,
+            "mean_first_page_identity_overlap": 0.94,
+            "minimum_first_page_identity_overlap": 0.41,
             "submission_difference_count": 0,
-            "count_difference_count": 2,
+            "count_difference_count": 3,
+            "maximum_count_delta": 4,
         }
-        differences = [
-            {"baseline_count": 3, "candidate_count": 4},
-            {"baseline_count": 3, "candidate_count": 4},
-        ]
 
-        gate = compare_reports.evaluate_parity(True, True, True, summary, differences)
+        gate = compare_reports.evaluate_parity(True, True, True, summary)
 
         self.assertTrue(gate["passed"])
-        self.assertEqual(gate["observed"]["maximum_count_delta"], 1)
+        self.assertEqual(gate["observed"]["maximum_count_delta"], 4)
+        self.assertTrue(gate["policy"]["result_counts_are_diagnostic_only"])
 
-    def test_rejects_tokenizer_style_count_explosion(self):
+    def test_rejects_widespread_low_result_overlap(self):
         summary = {
             "compared_probe_count": 284,
             "exact_first_page_parity_count": 109,
+            "substantial_first_page_overlap_count": 200,
+            "mean_first_page_identity_overlap": 0.42,
+            "minimum_first_page_identity_overlap": 0.0,
             "submission_difference_count": 0,
             "count_difference_count": 170,
+            "maximum_count_delta": 9999,
         }
-        differences = [{"baseline_count": 1, "candidate_count": 10000}]
 
-        gate = compare_reports.evaluate_parity(True, True, True, summary, differences)
+        gate = compare_reports.evaluate_parity(True, True, True, summary)
 
         self.assertFalse(gate["passed"])
-        self.assertFalse(gate["checks"]["maximum_count_delta_at_most_1"])
+        self.assertFalse(
+            gate["checks"][
+                "substantial_first_page_overlap_ratio_at_least_90_percent"
+            ]
+        )
+
+    def test_jaccard_overlap_uses_result_identity(self):
+        left = {"/one", "/two", "/three"}
+        right = {"/two", "/three", "/four"}
+
+        self.assertAlmostEqual(compare_reports.jaccard_overlap(left, right), 0.5)
+        self.assertEqual(compare_reports.jaccard_overlap(set(), set()), 1.0)
+
+    def test_result_identities_ignore_rendered_metadata(self):
+        results = [
+            {"uri_path": "/shared", "name": "Explorer name"},
+            {"uri_path": "/shared", "name": "sbol-db name"},
+            {"uri_path": None, "name": "not an addressable result"},
+        ]
+
+        self.assertEqual(compare_reports.result_identities(results), {"/shared"})
+
+    def test_report_comparison_gates_on_overlap_not_count_equality(self):
+        corpus = {
+            "revision": "pinned",
+            "discovered_xml_documents": 1,
+            "selected_xml_documents": 1,
+            "manifest_sha256": "same",
+        }
+        baseline = {
+            "topology": "explorer",
+            "corpus": corpus,
+            "submissions": [{"path": "probe.xml", "status": 200}],
+            "probes": [
+                {
+                    "path": "probe.xml",
+                    "count": 88,
+                    "results": [
+                        {"uri_path": "/one"},
+                        {"uri_path": "/two"},
+                        {"uri_path": "/three"},
+                    ],
+                }
+            ],
+            "gate": {"passed": True},
+        }
+        candidate = {
+            **baseline,
+            "topology": "sbol-db",
+            "probes": [
+                {
+                    "path": "probe.xml",
+                    "count": 84,
+                    "results": [
+                        {"uri_path": "/two", "name": "different metadata"},
+                        {"uri_path": "/three"},
+                        {"uri_path": "/four"},
+                    ],
+                }
+            ],
+        }
+
+        with tempfile.TemporaryDirectory() as directory:
+            baseline_path = Path(directory) / "baseline.json"
+            candidate_path = Path(directory) / "candidate.json"
+            output_path = Path(directory) / "comparison.json"
+            baseline_path.write_text(json.dumps(baseline))
+            candidate_path.write_text(json.dumps(candidate))
+
+            completed = subprocess.run(
+                [
+                    sys.executable,
+                    str(MODULE_PATH),
+                    "--baseline",
+                    str(baseline_path),
+                    "--candidate",
+                    str(candidate_path),
+                    "--output",
+                    str(output_path),
+                ],
+                check=False,
+                capture_output=True,
+                text=True,
+            )
+            comparison = json.loads(output_path.read_text())
+
+        self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr)
+        self.assertTrue(comparison["passed"])
+        self.assertEqual(comparison["schema_version"], 4)
+        self.assertEqual(comparison["summary"]["count_difference_count"], 1)
+        self.assertEqual(
+            comparison["probe_differences"][0]["first_page_identity_overlap"],
+            0.5,
+        )
 
 
 if __name__ == "__main__":

tests/test_explorer_search.py

@@ -1,17 +1,16 @@
 from unittest import TestCase
 
 from test_arguments import test_print
-from test_functions import compare_get_request, refresh_explorer_index
+from test_functions import compare_search_result_overlap, refresh_explorer_index
 
 
 class TestExplorerSearch(TestCase):
 
-    def test_indexed_search_html(self):
-        """Index the fixture explicitly, then enforce SBH1's rendered result."""
+    def test_indexed_search_overlap(self):
+        """Index the fixture, then require semantic result identity overlap."""
         test_print("test_explorer_indexed_search starting")
         refresh_explorer_index("I0462", "BBa_I0462")
-        # Reuse the established whole-page snapshot. Explorer-enabled matrix
-        # rows run this focused contract instead of TestSearch, so the request
-        # path remains unique in the legacy TestState registry.
-        compare_get_request("/search/:query?", route_parameters=["I0462"])
+        compare_search_result_overlap(
+            "/search/:query?", route_parameters=["I0462"]
+        )
         test_print("test_explorer_indexed_search completed")