Add compositional analysis code and example

examples/compositional_analysis/README.md

@@ -0,0 +1,14 @@
+# Compositional Analysis for Markovian Specifications
+
+This directory provides an example use case of **compositional analysis** for *Markovian* (i.e., memoryless) specifications.
+
+The core idea is to perform **Statistical Model Checking (SMC)** or **falsification** on *primitive scenarios* — scenarios that serve as building blocks for defining more complex *composite scenarios*. The analysis traces generated from these primitives are stored in a `ScenarioBase` object.
+
+These traces can then be supplied to an instance of `CompositionalAnalysisEngine`, which supports querying over composite scenario structures to perform compositional analysis based on the primitive scenario traces.
+
+This example uses the [MetaDrive](https://metadriverse.github.io/metadrive/) simulator.
+
+* Train a reinforcement learning policy using `train.py`.
+* Test the policy and save generated traces using `test.py`.
+* See `analyze.py` for an example of how to perform compositional analysis on generated traces.
+

examples/compositional_analysis/analyze.py

@@ -0,0 +1,47 @@
+import pandas as pd
+from verifai.compositional_analysis import CompositionalAnalysisEngine, ScenarioBase
+
+
+if __name__ == "__main__":
+    logs = {
+        "S": "storage/traces/S/traces.csv",
+        "X": "storage/traces/X/traces.csv",
+        # "O": "storage/traces/O/traces.csv",
+        # "C": "storage/traces/C/traces.csv",
+        "SX": "storage/traces/SX/traces.csv",
+        # "SO": "storage/traces/SO/traces.csv",
+        # "SC": "storage/traces/SC/traces.csv",
+        "SXS": "storage/traces/SXS/traces.csv",
+        # "SOS": "storage/traces/SOS/traces.csv",
+        # "SCS": "storage/traces/SCS/traces.csv",
+    }
+    scenario_base = ScenarioBase(logs, delta=0.01)
+
+    print("SMC")
+    for s in logs:
+        print(f"{s}: rho = {scenario_base.get_success_prob(s):.4f} ± {scenario_base.get_success_prob_uncertainty(s):.4f}")
+
+    engine = CompositionalAnalysisEngine(scenario_base)
+
+    pd.set_option('display.max_rows', None)  # Display all rows
+    pd.set_option('display.max_columns', None) # Display all columns
+    pd.set_option('display.width', 1000) # Ensure enough width to prevent wrapping
+
+    for s in logs:
+        print(f"Scenario: {s}")
+        print(f"Compositional SMC")
+        rho, uncertainty = engine.check(
+            s,
+            features=["x", "y", "heading", "speed"],
+            center_feat_idx=[0, 1],
+        )
+        print(f"Estimated {s}: rho = {rho:.4f} ± {uncertainty:.4f}")
+        print(f"Compositional Falsification")
+        cex = engine.falsify(
+            s,
+            features=["x", "y", "heading", "speed"],
+            center_feat_idx=[0, 1],
+            align_feat_idx=[0, 1],
+        )
+        print(f"Counterexample = {cex}")
+

examples/compositional_analysis/exps.sh

@@ -0,0 +1,23 @@
+# Fixed Error (epsilon = 0.1)
+## Monolithic
+{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_SX --confidence_level 0.95 --error_bound 0.04295 --scenario "SX"; } &> storage/results/fixed_error_monolithic_SX.txt &
+{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_COS --confidence_level 0.95 --error_bound 0.04295 --scenario "COS"; } &> storage/results/fixed_error_monolithic_COS.txt &
+{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_XSOC --confidence_level 0.95 --error_bound 0.04295 --scenario "XSOC"; } &> storage/results/fixed_error_monolithic_XSOC.txt &
+{ time python run_exp.py --expert --save_dir storage/fixed_error_monolithic_SOCXS --confidence_level 0.95 --error_bound 0.04295 --scenario "SOCXS"; } &> storage/results/fixed_error_monolithic_SOCXS.txt &
+## Compositional
+{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_SX --confidence_level 0.95 --error_bound 0.04295 --scenario "SX" --compositional; } &> storage/results/fixed_error_compositional_SX.txt &
+{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_COS --confidence_level 0.95 --error_bound 0.04295 --scenario "COS" --compositional; } &> storage/results/fixed_error_compositional_COS.txt &
+{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_XSOC --confidence_level 0.95 --error_bound 0.04295 --scenario "XSOC" --compositional; } &> storage/results/fixed_error_compositional_XSOC.txt &
+{ time python run_exp.py --expert --save_dir storage/fixed_error_compositional_SOCXS --confidence_level 0.95 --error_bound 0.04295 --scenario "SOCXS" --compositional; } &> storage/results/fixed_error_compositional_SOCXS.txt &
+
+# Fixed Time Budget (2 mins)
+## Monolithic
+{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic_SX --confidence_level 0.95 --time_budget 120 --scenario "SX"; } &> storage/results/fixed_time_monolithic_SX.txt &
+{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic_COS --confidence_level 0.95 --time_budget 120 --scenario "COS"; } &> storage/results/fixed_time_monolithic_COS.txt &
+{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic_XSOC --confidence_level 0.95 --time_budget 120 --scenario "XSOC"; } &> storage/results/fixed_time_monolithic_XSOC.txt &
+{ time python run_exp.py --expert --save_dir storage/fixed_time_monolithic_SOCXS --confidence_level 0.95 --time_budget 120 --scenario "SOCXS"; } &> storage/results/fixed_time_monolithic_SOCXS.txt &
+## Compositional
+{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional_SX --confidence_level 0.95 --time_budget 120 --scenario "SX" --compositional; } &> storage/results/fixed_time_compositional_SX.txt &
+{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional_COS --confidence_level 0.95 --time_budget 120 --scenario "COS" --compositional; } &> storage/results/fixed_time_compositional_COS.txt &
+{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional_XSOC --confidence_level 0.95 --time_budget 120 --scenario "XSOC" --compositional; } &> storage/results/fixed_time_compositional_XSOC.txt &
+{ time python run_exp.py --expert --save_dir storage/fixed_time_compositional_SOCXS --confidence_level 0.95 --time_budget 120 --scenario "SOCXS" --compositional; } &> storage/results/fixed_time_compositional_SOCXS.txt &