test: add source-backed IQ and MEX conformance baseline

.github/workflows/ci.yml

@@ -10,6 +10,21 @@ permissions:
   contents: read
 
 jobs:
+  conformance-fixtures:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: '3.x'
+      # Uses only the standard library and committed fixtures. No bundle restore
+      # or compiled extractor is needed for integrity and rejection checks.
+      - name: Verify conformance fixture integrity and capture rejection
+        run: python3 tests/conformance/test_capture.py -v
+
   check:
     runs-on: ubuntu-latest
     timeout-minutes: 20
@@ -35,6 +50,9 @@ jobs:
       - name: Test
         run: cargo test --workspace
 
+      - name: Test conformance capture parser
+        run: cargo test -p wa-transform --example module_spans
+
       # The JSON Schema derivation + envelope self-tests only build under the
       # `schema` feature, so run them explicitly.
       - name: Schema self-tests

crates/wa-mex/tests/independent_conformance.rs

@@ -0,0 +1,321 @@
+//! Bounded input-variable triage, not response-selection or complete MEX coverage.
+//! Presence is a serialization property, separate from inferred type/nullability.
+use std::{collections::BTreeMap, path::Path};
+use wa_ir::{MexOperationKind, TypeNode, VariablePresence, VariablePresenceNode};
+
+fn source(version: &str, module: &str) -> String {
+    std::fs::read_to_string(
+        Path::new(env!("CARGO_MANIFEST_DIR"))
+            .join("../../tests/conformance")
+            .join(version)
+            .join(format!("{module}.js")),
+    )
+    .unwrap()
+}
+
+#[test]
+fn verified_call_sites_keep_presence_separate_from_boolean_type() {
+    for version in ["2.3000.1045368834", "2.3000.1047483476"] {
+        let source = source(version, "WAWebMexFetchNewsletterJobQuery.graphql")
+            + &source(version, "WAWebMexFetchNewsletterJob");
+        let ir = wa_mex::extract_mex(&source, version);
+        reviewed_catalog(&ir);
+    }
+}
+
+fn reviewed_catalog(ir: &wa_ir::MexIr) {
+    assert_eq!(
+        ir.operations.keys().map(String::as_str).collect::<Vec<_>>(),
+        ["FetchNewsletter"]
+    );
+    let op = &ir.operations["FetchNewsletter"];
+    // Reviewed from the preserved Relay params literal, not generated output.
+    assert_eq!(op.original_name, "WAWebMexFetchNewsletterJobQuery");
+    assert_eq!(op.operation_kind, MexOperationKind::Query);
+    assert_eq!(op.doc_id, "27456920720571478");
+    reviewed_presence(op);
+}
+
+fn reviewed_presence(op: &wa_ir::MexOperation) {
+    // The source fixes object structure and argument names. Scalar tags remain
+    // the documented name-based approximation, not GraphQL/server type proof.
+    let boolean = || TypeNode::Leaf("boolean".into());
+    let string = || TypeNode::Leaf("string".into());
+    assert_eq!(
+        op.variables_shape,
+        BTreeMap::from([
+            ("fetch_creation_time".into(), boolean()),
+            ("fetch_full_image".into(), boolean()),
+            ("fetch_status_metadata".into(), boolean()),
+            ("fetch_wamo_sub".into(), boolean()),
+            ("fetch_viewer_metadata".into(), boolean()),
+            ("fetch_pinned_messages".into(), boolean()),
+            (
+                "input".into(),
+                TypeNode::Object(BTreeMap::from([
+                    ("key".into(), string()),
+                    ("type".into(), string()),
+                    ("view_role".into(), string()),
+                ]))
+            ),
+        ])
+    );
+    let leaf = |presence| VariablePresenceNode {
+        presence,
+        fields: BTreeMap::new(),
+        items: None,
+    };
+    assert_eq!(
+        op.variables_presence,
+        BTreeMap::from([
+            ("fetch_creation_time".into(), leaf(VariablePresence::Always)),
+            ("fetch_full_image".into(), leaf(VariablePresence::Always)),
+            (
+                "fetch_status_metadata".into(),
+                leaf(VariablePresence::Always)
+            ),
+            ("fetch_wamo_sub".into(), leaf(VariablePresence::Always)),
+            (
+                "fetch_viewer_metadata".into(),
+                leaf(VariablePresence::Conditional)
+            ),
+            (
+                "fetch_pinned_messages".into(),
+                leaf(VariablePresence::Undetermined)
+            ),
+            (
+                "input".into(),
+                VariablePresenceNode {
+                    presence: VariablePresence::Always,
+                    fields: BTreeMap::from([
+                        ("key".into(), leaf(VariablePresence::Conditional)),
+                        ("type".into(), leaf(VariablePresence::Always)),
+                        ("view_role".into(), leaf(VariablePresence::Conditional)),
+                    ]),
+                    items: None,
+                }
+            ),
+        ])
+    );
+}
+
+#[test]
+fn presence_oracle_rejects_regressions_in_all_reviewed_keys() {
+    let source = source(
+        "2.3000.1047483476",
+        "WAWebMexFetchNewsletterJobQuery.graphql",
+    ) + &source("2.3000.1047483476", "WAWebMexFetchNewsletterJob");
+    let ir = wa_mex::extract_mex(&source, "fixture");
+    for key in ["fetch_status_metadata", "fetch_wamo_sub", "view_role"] {
+        let mut op = ir.operations["FetchNewsletter"].clone();
+        if key == "view_role" {
+            op.variables_presence
+                .get_mut("input")
+                .unwrap()
+                .fields
+                .get_mut(key)
+                .unwrap()
+                .presence = VariablePresence::Always;
+        } else {
+            op.variables_presence.get_mut(key).unwrap().presence = VariablePresence::Undetermined;
+        }
+        assert!(
+            std::panic::catch_unwind(|| reviewed_presence(&op)).is_err(),
+            "accepted {key} regression"
+        );
+    }
+}
+
+#[test]
+fn null_undefined_and_unresolved_calls_have_distinct_presence_verdicts() {
+    // Hand-authored perturbations of a real operation, not extra claims about
+    // which arguments the server accepts. Expectations follow JSON object-key
+    // serialization: null survives, undefined disappears, calls are unresolved.
+    let query = source(
+        "2.3000.1047483476",
+        "WAWebMexFetchNewsletterJobQuery.graphql",
+    );
+    for (expression, expected) in [
+        ("null", VariablePresence::Always),
+        ("false", VariablePresence::Always),
+        ("void 0", VariablePresence::Conditional),
+        ("t.flag", VariablePresence::Conditional),
+        ("t.flag === true", VariablePresence::Always),
+        ("o(\"External\").flag()", VariablePresence::Undetermined),
+    ] {
+        let caller = format!(
+            r#"__d("ConformanceCaller",["WAWebMexClient","WAWebMexFetchNewsletterJobQuery.graphql","External"],(function(t,n,r,o,a,i,l){{function f(t){{return o("WAWebMexClient").fetchQuery(n("WAWebMexFetchNewsletterJobQuery.graphql"),{{fetch_viewer_metadata:{expression}}})}}l.run=f}}),98);"#
+        );
+        let ir = wa_mex::extract_mex(&(query.clone() + &caller), "fixture");
+        let op = &ir.operations["FetchNewsletter"];
+        assert_eq!(
+            op.variables_presence["fetch_viewer_metadata"].presence, expected,
+            "{expression}"
+        );
+        assert_eq!(
+            op.variables_shape["fetch_viewer_metadata"],
+            TypeNode::Leaf("boolean".into()),
+            "presence does not establish nullability: {expression}"
+        );
+    }
+}
+
+#[test]
+fn presence_oracle_rejects_unreviewed_keys() {
+    let source = source(
+        "2.3000.1047483476",
+        "WAWebMexFetchNewsletterJobQuery.graphql",
+    ) + &source("2.3000.1047483476", "WAWebMexFetchNewsletterJob");
+    let ir = wa_mex::extract_mex(&source, "fixture");
+    for nested in [false, true] {
+        let mut op = ir.operations["FetchNewsletter"].clone();
+        if nested {
+            let input = op.variables_presence.get_mut("input").unwrap();
+            input
+                .fields
+                .insert("unreviewed".into(), input.fields["key"].clone());
+        } else {
+            op.variables_presence
+                .insert("unreviewed".into(), op.variables_presence["input"].clone());
+        }
+        assert!(
+            std::panic::catch_unwind(|| reviewed_presence(&op)).is_err(),
+            "accepted extra key, nested={nested}"
+        );
+    }
+}
+
+#[test]
+fn source_catalog_rejects_spurious_mex_operations() {
+    for version in ["2.3000.1045368834", "2.3000.1047483476"] {
+        let source = source(version, "WAWebMexFetchNewsletterJobQuery.graphql")
+            + &source(version, "WAWebMexFetchNewsletterJob");
+        let mut ir = wa_mex::extract_mex(&source, version);
+        ir.operations.insert(
+            "Unreviewed".into(),
+            ir.operations["FetchNewsletter"].clone(),
+        );
+        assert!(std::panic::catch_unwind(|| reviewed_catalog(&ir)).is_err());
+    }
+}
+
+#[test]
+fn variables_shape_oracle_rejects_missing_extra_and_changed_fields() {
+    let source = source(
+        "2.3000.1047483476",
+        "WAWebMexFetchNewsletterJobQuery.graphql",
+    ) + &source("2.3000.1047483476", "WAWebMexFetchNewsletterJob");
+    let ir = wa_mex::extract_mex(&source, "fixture");
+    let original = &ir.operations["FetchNewsletter"];
+    for nested in [false, true] {
+        let keys: Vec<_> = if nested {
+            vec!["key", "type", "view_role"]
+        } else {
+            vec![
+                "input",
+                "fetch_creation_time",
+                "fetch_full_image",
+                "fetch_status_metadata",
+                "fetch_wamo_sub",
+                "fetch_viewer_metadata",
+                "fetch_pinned_messages",
+            ]
+        };
+        for key in keys.into_iter().chain(["unreviewed"]) {
+            for remove in [false, true] {
+                if key == "unreviewed" && remove {
+                    continue;
+                }
+                let mut op = original.clone();
+                let fields = if nested {
+                    let TypeNode::Object(fields) = op.variables_shape.get_mut("input").unwrap()
+                    else {
+                        panic!("input must be an object")
+                    };
+                    fields
+                } else {
+                    &mut op.variables_shape
+                };
+                if remove {
+                    fields.remove(key);
+                } else {
+                    fields.insert(key.into(), TypeNode::Leaf("wrong".into()));
+                }
+                assert!(
+                    std::panic::catch_unwind(|| reviewed_presence(&op)).is_err(),
+                    "accepted shape mutation {key}, nested={nested}, remove={remove}"
+                );
+            }
+        }
+    }
+}
+
+#[test]
+fn presence_oracle_rejects_children_and_items_on_scalar_leaves() {
+    let source = source(
+        "2.3000.1047483476",
+        "WAWebMexFetchNewsletterJobQuery.graphql",
+    ) + &source("2.3000.1047483476", "WAWebMexFetchNewsletterJob");
+    let ir = wa_mex::extract_mex(&source, "fixture");
+    for path in [
+        "fetch_creation_time",
+        "fetch_full_image",
+        "fetch_status_metadata",
+        "fetch_wamo_sub",
+        "fetch_viewer_metadata",
+        "fetch_pinned_messages",
+        "key",
+        "type",
+        "view_role",
+        "input",
+    ] {
+        for items in [false, true] {
+            if path == "input" && !items {
+                continue;
+            }
+            let mut op = ir.operations["FetchNewsletter"].clone();
+            let extra = op.variables_presence["fetch_full_image"].clone();
+            let node = if ["key", "type", "view_role"].contains(&path) {
+                op.variables_presence
+                    .get_mut("input")
+                    .unwrap()
+                    .fields
+                    .get_mut(path)
+                    .unwrap()
+            } else {
+                op.variables_presence.get_mut(path).unwrap()
+            };
+            if items {
+                node.items = Some(Box::new(extra));
+            } else {
+                node.fields.insert("unreviewed".into(), extra);
+            }
+            assert!(
+                std::panic::catch_unwind(|| reviewed_presence(&op)).is_err(),
+                "accepted presence structure {path}, items={items}"
+            );
+        }
+    }
+}
+
+#[test]
+fn source_catalog_rejects_changed_operation_identity() {
+    let source = source(
+        "2.3000.1047483476",
+        "WAWebMexFetchNewsletterJobQuery.graphql",
+    ) + &source("2.3000.1047483476", "WAWebMexFetchNewsletterJob");
+    let original = wa_mex::extract_mex(&source, "fixture");
+    for field in ["name", "kind", "doc_id"] {
+        let mut ir = original.clone();
+        let op = ir.operations.get_mut("FetchNewsletter").unwrap();
+        match field {
+            "name" => op.original_name = "Unreviewed".into(),
+            "kind" => op.operation_kind = MexOperationKind::Mutation,
+            _ => op.doc_id = "0".into(),
+        }
+        assert!(
+            std::panic::catch_unwind(|| reviewed_catalog(&ir)).is_err(),
+            "accepted changed operation {field}"
+        );
+    }
+}