[vastly] multi sim compare

.github/workflows/ci.yml

@@ -103,9 +103,9 @@ jobs:
         run: cargo test --test version_test
 
   # -----------------------------------------------------------
-  # 2) xlsynth-vastly optional oracle lane (iverilog/vvp)
+  # 2) xlsynth-vastly reference simulator lane (currently Icarus)
   # -----------------------------------------------------------
-  xlsynth-vastly-iverilog-tests:
+  xlsynth-vastly-reference-sim-tests:
     needs: [lint-check-ubuntu]
     runs-on: ubuntu-latest
 
@@ -130,16 +130,16 @@ jobs:
           profile: minimal
           override: true
 
-      - name: Install Icarus Verilog
+      - name: Install Icarus Verilog reference simulator
         run: |
           sudo apt-get update
           sudo apt-get install -y iverilog
           iverilog -V
           vvp -V
 
-      - name: Run xlsynth-vastly oracle tests
+      - name: Run xlsynth-vastly reference simulator tests
         run: |
-          cargo test -p xlsynth-vastly --features iverilog-tests
+          cargo test -p xlsynth-vastly --features reference-sim-tests
 
   # -----------------------------------------------------------
   # 3) Build/test on Ubuntu 22.04 and 24.04

xlsynth-vastly/Cargo.toml

@@ -22,4 +22,4 @@ xlsynth = { path = "../xlsynth", version = "0.36.0" }
 
 [features]
 default = []
-iverilog-tests = []
+reference-sim-tests = []

xlsynth-vastly/README.md

@@ -3,8 +3,28 @@
 `xlsynth-vastly` simulates Verilog/SystemVerilog that XLS codegen emits, with
 support for both combinational (`combo`) and pipelined designs over the
 constructs we target. We ground correctness with fuzzing over XLS-generated
-artifacts in both forms, and with oracle-style comparisons such as semantic VCD
-diffing against `iverilog`/`vvp`.
+artifacts in both forms, and with differential checks against external
+reference implementations such as semantic VCD diffing against an external
+simulator backend.
+
+The language standards are the semantic source of truth here. External
+simulators are used as independent implementations to cross-check against the
+spec, not as spec providers themselves.
+
+## Scope and Trust Model
+
+`xlsynth-vastly` is a deliberately limited simulator intended for testing and
+debugging the subset of Verilog/SystemVerilog emitted by XLS and VAST.
+
+It is not designed or intended to be a general-purpose Verilog simulator. The
+implementation makes a best-effort attempt to reflect the Verilog/SystemVerilog
+specification, but any trust in its behavior should be limited to the language
+subset exercised by generated XLS/VAST outputs and the surrounding regression
+and differential-test coverage.
+
+In other words: if `xlsynth-vastly` accepts and correctly simulates broader
+source-language inputs, that is useful, but it is not the primary contract of
+the crate.
 
 ## Quickstart Commands
 
@@ -27,13 +47,13 @@ cargo run -p xlsynth-vastly --bin vastly-sim-pipeline -- /path/to/foo.sv \
   --vcd-out ./pipeline.vcd
 ```
 
-- Compare combo simulation results with Icarus Verilog (best as a semantic oracle check):
+- Compare combo simulation results against a reference simulator:
 
 ```bash
 cargo run -p xlsynth-vastly --bin vastly-sim-combo -- /path/to/foo.combo.v \
   --inputs "0xf00,0xba5;0x0000,0x3f80" \
   --vcd-out ./combo.vcd \
-  --compare-to-iverilog
+  --compare-to-reference-sim=iverilog
 ```
 
 - Run baseline tests that do not require external simulators:
@@ -42,23 +62,23 @@ cargo run -p xlsynth-vastly --bin vastly-sim-combo -- /path/to/foo.combo.v \
 cargo test -p xlsynth-vastly
 ```
 
-- Run extended oracle tests that use `iverilog`/`vvp`:
+- Run extended reference-simulator tests:
 
 ```bash
-cargo test -p xlsynth-vastly --features iverilog-tests
+cargo test -p xlsynth-vastly --features reference-sim-tests
 ```
 
 ## Tooling Requirements
 
-- Core unit/integration tests do not require `iverilog`/`vvp`.
-- Oracle/differential tests against Icarus Verilog require `iverilog` and `vvp`
-  on `PATH`, and are enabled with `--features iverilog-tests`.
-- `iverilog-tests` exists so default `cargo test` remains tool-independent,
-  while still providing an explicit opt-in lane for oracle-backed comparisons.
-- In `iverilog-tests` mode, `iverilog`/`vvp` are only invoked as subprocesses to
-  produce oracle outputs; the oracle backend is intentionally pluggable so
-  additional simulators can be slotted in later (e.g. for further SV support
-  comparisons).
+- Core unit/integration tests do not require external simulators.
+- Reference-simulator tests require the current external simulator backend to
+  be available on `PATH`; enable them with `--features reference-sim-tests`.
+- `reference-sim-tests` exists so default `cargo test` remains
+  tool-independent while still providing an explicit opt-in lane for
+  differential checks against external implementations.
+- The reference-simulator layer is intentionally pluggable so additional
+  implementations can be slotted in later, including backends with only
+  two-value semantics such as Yosys/CXXRTL.
 
 ## Stimulus Format
 
@@ -96,15 +116,16 @@ cargo run -p xlsynth-vastly --bin vastly-sim-combo -- /path/to/foo.combo.v \
 - Values may be hex (`0x...`) or decimal.
 - Values map positionally to module input ports in module-header order.
 
-### Compare against Icarus Verilog
+### Compare against a reference simulator
 
-If `iverilog`/`vvp` are on `PATH`, generate an Icarus VCD and semantically diff:
+If the current reference-simulator backend is on `PATH`, generate a reference
+VCD and semantically diff:
 
 ```bash
 cargo run -p xlsynth-vastly --bin vastly-sim-combo -- /path/to/foo.combo.v \
   --inputs "0xf00,0xba5;0x0000,0x3f80" \
   --vcd-out ./combo.vcd \
-  --compare-to-iverilog
+  --compare-to-reference-sim=iverilog
 ```
 
 ## `vastly-sim-pipeline`
@@ -130,8 +151,8 @@ cargo run -p xlsynth-vastly --bin vastly-sim-pipeline -- /path/to/foo.sv \
 
 ## Testing
 
-- Baseline tests (no Icarus dependency):
+- Baseline tests (no external-simulator dependency):
   `cargo test -p xlsynth-vastly`
-- Extended oracle tests (requires `iverilog`/`vvp`):
-  `cargo test -p xlsynth-vastly --features iverilog-tests`
+- Extended reference-simulator tests:
+  `cargo test -p xlsynth-vastly --features reference-sim-tests`
 - Fuzz targets live under `xlsynth-vastly/fuzz`; see workspace `FUZZ.md`.

xlsynth-vastly/fuzz/Cargo.toml

@@ -28,8 +28,8 @@ members = ["."]
 warnings = "deny"
 
 [[bin]]
-name = "diff_iverilog"
-path = "fuzz_targets/diff_iverilog.rs"
+name = "diff_refsim_4value"
+path = "fuzz_targets/diff_refsim_4value.rs"
 test = false
 doc = false
 

xlsynth-vastly/fuzz/fuzz_targets/xls_ir_codegen_semantics.rs

@@ -237,11 +237,12 @@ fuzz_target!(|data: &[u8]| {
             stimulus,
             &stimulus_ordinal,
         );
-        if include_iverilog_oracle() {
+        if include_iverilog_reference_sim() {
             // Keep Icarus on the plain-Verilog path only. XLS can emit
             // SystemVerilog array assignment forms that current Icarus does not
-            // support, so the external oracle is: Icarus Verilog, plus an
-            // explicit Vastly-V/Vastly-SV equality check.
+            // support, so this reference implementation check is: Icarus
+            // Verilog on the plain-Verilog path, plus an explicit
+            // Vastly-V/Vastly-SV equality check.
             let got_iv_v = match eval_codegen_with_iverilog_verilog(&verilog_src, &input_map) {
                 Ok(v) => v,
                 Err(e) => {
@@ -290,6 +291,59 @@ fuzz_target!(|data: &[u8]| {
                 &stimulus_ordinal,
             );
         }
+        if include_yosys_cxxrtl_reference_sim() && input_map_is_two_value_safe(&input_map) {
+            let got_cxxrtl_v = match eval_codegen_with_yosys_cxxrtl_verilog(
+                &verilog_src,
+                "fuzz_codegen_v",
+                &input_map,
+            ) {
+                Ok(v) => v,
+                Err(e) => {
+                    unsupported(
+                        "yosys/cxxrtl could not compile/eval generated Verilog",
+                        &ir_text,
+                        Some(&verilog_src),
+                        Some(&format!(
+                            "{e} {}",
+                            summarize_stimulus_with_index(stimulus, &stimulus_ordinal)
+                        )),
+                        data,
+                    );
+                    return;
+                }
+            };
+
+            assert_same_bits(
+                "yosys_cxxrtl_verilog",
+                &want_bits,
+                &got_cxxrtl_v,
+                &ir_text,
+                &verilog_src,
+                data,
+                stimulus,
+                &stimulus_ordinal,
+            );
+            assert_same_bits(
+                "yosys_cxxrtl_v_vs_vastly_v",
+                &got_cxxrtl_v,
+                &got_v,
+                &ir_text,
+                &verilog_src,
+                data,
+                stimulus,
+                &stimulus_ordinal,
+            );
+            assert_same_bits(
+                "yosys_cxxrtl_v_vs_vastly_sv",
+                &got_cxxrtl_v,
+                &got_sv,
+                &ir_text,
+                &sv_src,
+                data,
+                stimulus,
+                &stimulus_ordinal,
+            );
+        }
     }
 
     let mut stage1_retired: Option<(String, Vec<Value4>)> = None;
@@ -609,6 +663,28 @@ fn eval_codegen_with_iverilog_verilog(
     result
 }
 
+fn eval_codegen_with_yosys_cxxrtl_verilog(
+    src: &str,
+    module_name: &str,
+    inputs: &BTreeMap<String, Value4>,
+) -> Result<Value4, String> {
+    let m = xlsynth_vastly::compile_combo_module(src)
+        .map_err(|e| format!("compile_combo_module failed before Yosys/CXXRTL run: {e:?}"))?;
+    if m.output_ports.len() != 1 {
+        return Err(format!(
+            "expected exactly one output port, got {}",
+            m.output_ports.len()
+        ));
+    }
+    let out_name = m.output_ports[0].name.clone();
+    let outputs = xlsynth_vastly::eval_yosys_cxxrtl_combo(src, module_name, inputs)
+        .map_err(|e| format!("eval_yosys_cxxrtl_combo failed: {e:?}"))?;
+    outputs
+        .get(&out_name)
+        .cloned()
+        .ok_or_else(|| format!("output `{out_name}` missing from Yosys/CXXRTL result"))
+}
+
 fn eval_pipeline_with_vastly(
     src: &str,
     pipeline_stages: usize,
@@ -809,8 +885,13 @@ fn strict_unsupported() -> bool {
     env_truthy("VASTLY_FUZZ_STRICT_UNSUPPORTED")
 }
 
-fn include_iverilog_oracle() -> bool {
-    env_truthy("VASTLY_FUZZ_INCLUDE_IVERILOG_ORACLE")
+fn include_iverilog_reference_sim() -> bool {
+    env_truthy("VASTLY_FUZZ_INCLUDE_IVERILOG_REFERENCE_SIM")
+        || env_truthy("VASTLY_FUZZ_INCLUDE_IVERILOG_ORACLE")
+}
+
+fn include_yosys_cxxrtl_reference_sim() -> bool {
+    env_truthy("VASTLY_FUZZ_INCLUDE_YOSYS_CXXRTL_REFERENCE_SIM")
 }
 
 fn include_stage1_pipeline_oracle() -> bool {
@@ -825,6 +906,14 @@ fn use_autocov_stimuli() -> bool {
     env_truthy("VASTLY_FUZZ_USE_AUTOCOV")
 }
 
+fn input_map_is_two_value_safe(input_map: &BTreeMap<String, Value4>) -> bool {
+    let mut env = xlsynth_vastly::Env::new();
+    for (name, value) in input_map {
+        env.insert(name.clone(), value.clone());
+    }
+    xlsynth_vastly::env_is_two_value_safe(&env)
+}
+
 fn unsupported(
     reason: &str,
     ir_text: &str,

xlsynth-vastly/fuzz/tests/codegen_semantics_typed_top_level.rs

@@ -5,6 +5,8 @@ use std::time::SystemTime;
 
 use xlsynth_vastly::compile_combo_module;
 use xlsynth_vastly::eval_combo;
+use xlsynth_vastly::eval_yosys_cxxrtl_combo;
+use xlsynth_vastly::has_yosys_cxxrtl_toolchain;
 use xlsynth_vastly::plan_combo_eval;
 use xlsynth_vastly::run_iverilog_combo_and_collect_vcd;
 use xlsynth_vastly::LogicBit;
@@ -140,6 +142,24 @@ fn eval_codegen_with_iverilog_verilog(
     result
 }
 
+fn eval_codegen_with_yosys_cxxrtl_verilog(
+    src: &str,
+    module_name: &str,
+    inputs: &BTreeMap<String, Value4>,
+) -> Result<Value4, String> {
+    let m = compile_combo_module(src).map_err(|e| format!("compile_combo_module failed: {e:?}"))?;
+    if m.output_ports.len() != 1 {
+        return Err(format!("expected exactly one output port, got {}", m.output_ports.len()));
+    }
+    let out_name = m.output_ports[0].name.clone();
+    let outputs = eval_yosys_cxxrtl_combo(src, module_name, inputs)
+        .map_err(|e| format!("eval_yosys_cxxrtl_combo failed: {e:?}"))?;
+    outputs
+        .get(&out_name)
+        .cloned()
+        .ok_or_else(|| format!("missing output `{out_name}`"))
+}
+
 fn value4_from_msb_bits(bits: &str) -> Result<Value4, String> {
     let mut out = Vec::with_capacity(bits.len());
     for c in bits.chars().rev() {
@@ -217,8 +237,22 @@ fn typed_top_level_codegen_matches_ir_and_oracles() {
     let got_v = eval_codegen_with_vastly(&verilog_src, &input_map).unwrap();
     let got_sv = eval_codegen_with_vastly(&sv_src, &input_map).unwrap();
     let got_iv = eval_codegen_with_iverilog_verilog(&verilog_src, &input_map).unwrap();
+    let got_cxxrtl = if has_yosys_cxxrtl_toolchain() {
+        Some(
+            eval_codegen_with_yosys_cxxrtl_verilog(&verilog_src, "compound_shapes_v", &input_map)
+                .unwrap(),
+        )
+    } else {
+        None
+    };
 
     assert_eq!(got_v.to_bit_string_msb_first(), want.to_bit_string_msb_first());
     assert_eq!(got_sv.to_bit_string_msb_first(), want.to_bit_string_msb_first());
     assert_eq!(got_iv.to_bit_string_msb_first(), want.to_bit_string_msb_first());
+    if let Some(got_cxxrtl) = got_cxxrtl {
+        assert_eq!(
+            got_cxxrtl.to_bit_string_msb_first(),
+            want.to_bit_string_msb_first()
+        );
+    }
 }