diff --git a/.gitignore b/.gitignore
index 354e485..8d0348e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -120,3 +120,5 @@ venv.bak/
dmypy.json
*.cproject
+
+examples/dynamic_rulebook/*/outputs/
diff --git a/examples/dynamic_rulebook/maps/Town05.net.xml b/examples/dynamic_rulebook/maps/Town05.net.xml
new file mode 100644
index 0000000..1769ec0
--- /dev/null
+++ b/examples/dynamic_rulebook/maps/Town05.net.xml
@@ -0,0 +1,3808 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/examples/dynamic_rulebook/maps/Town05.xodr b/examples/dynamic_rulebook/maps/Town05.xodr
new file mode 100644
index 0000000..88d0a31
--- /dev/null
+++ b/examples/dynamic_rulebook/maps/Town05.xodr
@@ -0,0 +1,47273 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi.py b/examples/dynamic_rulebook/multi.py
new file mode 100644
index 0000000..79bd91f
--- /dev/null
+++ b/examples/dynamic_rulebook/multi.py
@@ -0,0 +1,140 @@
+"""
+Framework for experimentation of multi-objective and dynamic falsification.
+
+Author: Kai-Chun Chang. Based on Kesav Viswanadha's code.
+"""
+
+import time
+import os
+import numpy as np
+from dotmap import DotMap
+import traceback
+import argparse
+import importlib
+
+from verifai.samplers.scenic_sampler import ScenicSampler
+from verifai.scenic_server import ScenicServer
+from verifai.falsifier import generic_falsifier, generic_parallel_falsifier
+from verifai.monitor import multi_objective_monitor, specification_monitor
+from verifai.rulebook import rulebook
+
+import networkx as nx
+import pandas as pd
+
+def announce(message):
+ lines = message.split('\n')
+ size = max([len(p) for p in lines]) + 4
+ def pad(line):
+ ret = '* ' + line
+ ret += ' ' * (size - len(ret) - 1) + '*'
+ return ret
+ lines = list(map(pad, lines))
+ m = '\n'.join(lines)
+ border = '*' * size
+ print(border)
+ print(m)
+ print(border)
+
+"""
+Runs all experiments in a directory.
+"""
+def run_experiments(path, rulebook=None, parallel=False, model=None,
+ sampler_type=None, headless=False, num_workers=5, output_dir='outputs',
+ experiment_name=None, max_time=None, n_iters=None, max_steps=300):
+ if not os.path.exists(output_dir):
+ os.mkdir(output_dir)
+ paths = []
+ if os.path.isdir(path):
+ for root, _, files in os.walk(path):
+ for name in files:
+ fname = os.path.join(root, name)
+ if os.path.splitext(fname)[1] == '.scenic':
+ paths.append(fname)
+ else:
+ paths = [path]
+ for p in paths:
+ falsifier = run_experiment(p, rulebook=rulebook,
+ parallel=parallel, model=model, sampler_type=sampler_type, headless=headless,
+ num_workers=num_workers, max_time=max_time, n_iters=n_iters, max_steps=max_steps)
+ df = pd.concat([falsifier.error_table.table, falsifier.safe_table.table])
+ if experiment_name is not None:
+ outfile = experiment_name
+ else:
+ root, _ = os.path.splitext(p)
+ outfile = root.split('/')[-1]
+ if parallel:
+ outfile += '_parallel'
+ if model:
+ outfile += f'_{model}'
+ if sampler_type:
+ outfile += f'_{sampler_type}'
+ outfile += '.csv'
+ outpath = os.path.join(output_dir, outfile)
+ print(f'(multi.py) Saving output to {outpath}')
+ df.to_csv(outpath)
+
+"""
+Runs a single falsification experiment.
+
+Arguments:
+ path: Path to Scenic script to be run.
+ parallel: Whether or not to enable parallelism.
+ model: Which simulator model to use (e.g. scenic.simulators.newtonian.driving_model)
+ sampler_type: Which VerifAI sampelr to use (e.g. halton, scenic, ce, mab, etc.)
+ headless: Whether or not to display each simulation.
+ num_workers: Number of parallel workers. Only used if parallel is true.
+"""
+def run_experiment(scenic_path, rulebook=None, parallel=False, model=None,
+ sampler_type=None, headless=False, num_workers=5, max_time=None,
+ n_iters=5, max_steps=300):
+ # Construct rulebook
+ rb = rulebook
+
+ # Construct sampler (scenic_sampler.py)
+ print(f'(multi.py) Running Scenic script {scenic_path}')
+ params = {'verifaiSamplerType': sampler_type} if sampler_type else {}
+ params['render'] = not headless
+ params['seed'] = 0
+ params['use2DMap'] = True
+ sampler = ScenicSampler.fromScenario(scenic_path, maxIterations=40000, params=params, model=model)
+ num_objectives = sampler.scenario.params.get('N', 1)
+ s_type = sampler.scenario.params.get('verifaiSamplerType', None)
+ print(f'(multi.py) num_objectives: {num_objectives}')
+
+ # Construct falsifier (falsifier.py)
+ multi = num_objectives > 1
+ falsifier_params = DotMap(
+ n_iters=n_iters,
+ save_error_table=True,
+ save_safe_table=True,
+ max_time=max_time,
+ verbosity=1,
+ )
+ server_options = DotMap(maxSteps=max_steps, verbosity=1,
+ scenic_path=scenic_path, scenario_params=params, scenario_model=model,
+ num_workers=num_workers)
+ falsifier_class = generic_parallel_falsifier if parallel else generic_falsifier
+ falsifier = falsifier_class(monitor=rb, ## modified
+ sampler_type=s_type,
+ sampler=sampler,
+ falsifier_params=falsifier_params,
+ server_options=server_options,
+ server_class=ScenicServer)
+ print(f'(multi.py) sampler_type: {falsifier.sampler_type}')
+
+ # Run falsification
+ t0 = time.time()
+ print('(multi.py) Running falsifier')
+ falsifier.run_falsifier()
+ t = time.time() - t0
+ print()
+ print(f'(multi.py) Generated {len(falsifier.samples)} samples in {t} seconds with {falsifier.num_workers} workers')
+ print(f'(multi.py) Number of counterexamples: {len(falsifier.error_table.table)}')
+ if not parallel:
+ print(f'(multi.py) Sampling time: {falsifier.total_sample_time}')
+ print(f'(multi.py) Simulation time: {falsifier.total_simulate_time}')
+ print(f'(multi.py) Confidence interval: {falsifier.get_confidence_interval()}')
+ return falsifier
+
+if __name__ == '__main__':
+ pass
diff --git a/examples/dynamic_rulebook/multi_01/multi_01.py b/examples/dynamic_rulebook/multi_01/multi_01.py
new file mode 100644
index 0000000..a4ba010
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_01/multi_01.py
@@ -0,0 +1,50 @@
+import sys
+import os
+sys.path.append(os.path.abspath("."))
+import random
+import numpy as np
+
+from multi import *
+from multi_01_rulebook import rulebook_multi01
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--scenic-path', '-sp', type=str, default='uberCrashNewton.scenic',
+ help='Path to Scenic script')
+ parser.add_argument('--graph-path', '-gp', type=str, default=None,
+ help='Path to graph file')
+ parser.add_argument('--rule-path', '-rp', type=str, default=None,
+ help='Path to rule file')
+ parser.add_argument('--output-dir', '-o', type=str, default=None,
+ help='Directory to save output trajectories')
+ parser.add_argument('--output-csv-dir', '-co', type=str, default=None,
+ help='Directory to save output error tables (csv files)')
+ parser.add_argument('--parallel', action='store_true')
+ parser.add_argument('--num-workers', type=int, default=5, help='Number of parallel workers')
+ parser.add_argument('--sampler-type', '-s', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--experiment-name', '-e', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--model', '-m', type=str, default='scenic.simulators.newtonian.driving_model')
+ parser.add_argument('--headless', action='store_true')
+ parser.add_argument('--n-iters', '-n', type=int, default=None, help='Number of simulations to run')
+ parser.add_argument('--max-time', type=int, default=None, help='Maximum amount of time to run simulations')
+ parser.add_argument('--single-graph', action='store_true', help='Only a unified priority graph')
+ parser.add_argument('--seed', type=int, default=0, help='Random seed')
+ parser.add_argument('--using-sampler', type=int, default=-1, help='Assigning sampler to use')
+ parser.add_argument('--exploration-ratio', type=float, default=2.0, help='Exploration ratio')
+ args = parser.parse_args()
+ if args.n_iters is None and args.max_time is None:
+ raise ValueError('At least one of --n-iters or --max-time must be set')
+
+ random.seed(args.seed)
+ np.random.seed(args.seed)
+
+ print('output_dir =', args.output_dir)
+ rb = rulebook_multi01(args.graph_path, args.rule_path, save_path=args.output_dir, single_graph=args.single_graph, using_sampler=args.using_sampler, exploration_ratio=args.exploration_ratio)
+ run_experiments(args.scenic_path, rulebook=rb,
+ parallel=args.parallel, model=args.model,
+ sampler_type=args.sampler_type, headless=args.headless,
+ num_workers=args.num_workers, output_dir=args.output_csv_dir, experiment_name=args.experiment_name,
+ max_time=args.max_time, n_iters=args.n_iters)
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_01/multi_01.scenic b/examples/dynamic_rulebook/multi_01/multi_01.scenic
new file mode 100644
index 0000000..fbabebe
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_01/multi_01.scenic
@@ -0,0 +1,93 @@
+"""
+TITLE: Multi 01
+AUTHOR: Kai-Chun Chang, kaichunchang@berkeley.edu
+DESCRIPTION: The ego vehicle is driving along its lane when it encounters a blocking car ahead. The ego attempts to change to the opposite lane to bypass the blocking car before returning to its original lane.
+"""
+
+#################################
+# MAP AND MODEL #
+#################################
+
+param map = localPath('../maps/Town05.xodr')
+param carla_map = 'Town05'
+param N = 2
+model scenic.domains.driving.model
+
+#################################
+# CONSTANTS #
+#################################
+
+MODEL = 'vehicle.lincoln.mkz_2017'
+
+param EGO_SPEED = VerifaiRange(6, 9) #7
+param DIST_THRESHOLD = VerifaiRange(12, 14) #13
+param BLOCKING_CAR_DIST = VerifaiRange(15, 20)
+param BYPASS_DIST = VerifaiRange(4, 6) #5
+
+DIST_TO_INTERSECTION = 15
+TERM_DIST = 40
+
+#################################
+# AGENT BEHAVIORS #
+#################################
+
+behavior EgoBehavior(path):
+ current_lane = network.laneAt(self)
+ laneChangeCompleted = False
+ bypassed = False
+ try:
+ do FollowLaneBehavior(globalParameters.EGO_SPEED, laneToFollow=current_lane)
+ interrupt when (distance to blockingCar) < globalParameters.DIST_THRESHOLD and not laneChangeCompleted:
+ do LaneChangeBehavior(path, is_oppositeTraffic=True, target_speed=globalParameters.EGO_SPEED)
+ do FollowLaneBehavior(globalParameters.EGO_SPEED, is_oppositeTraffic=True) until (distance to blockingCar) > globalParameters.BYPASS_DIST
+ laneChangeCompleted = True
+ interrupt when (blockingCar can see ego) and (distance to blockingCar) > globalParameters.BYPASS_DIST and not bypassed:
+ current_laneSection = network.laneSectionAt(self)
+ rightLaneSec = current_laneSection._laneToLeft
+ do LaneChangeBehavior(rightLaneSec, is_oppositeTraffic=False, target_speed=globalParameters.EGO_SPEED)
+ bypassed = True
+
+#################################
+# SPATIAL RELATIONS #
+#################################
+
+#Find lanes that have a lane to their left in the opposite direction
+laneSecsWithLeftLane = []
+for lane in network.lanes:
+ for laneSec in lane.sections:
+ if laneSec._laneToLeft is not None:
+ if laneSec._laneToLeft.isForward is not laneSec.isForward:
+ laneSecsWithLeftLane.append(laneSec)
+
+assert len(laneSecsWithLeftLane) > 0, \
+ 'No lane sections with adjacent left lane with opposing \
+ traffic direction in network.'
+
+initLaneSec = Uniform(*laneSecsWithLeftLane)
+leftLaneSec = initLaneSec._laneToLeft
+
+spawnPt = new OrientedPoint on initLaneSec.centerline
+
+#################################
+# SCENARIO SPECIFICATION #
+#################################
+
+ego = new Car at spawnPt,
+ with blueprint MODEL,
+ with behavior EgoBehavior(leftLaneSec)
+
+blockingCar = new Car following roadDirection from ego for globalParameters.BLOCKING_CAR_DIST,
+ with blueprint MODEL,
+ with viewAngle 90 deg
+
+require (distance from blockingCar to intersection) > DIST_TO_INTERSECTION
+terminate when (distance to spawnPt) > TERM_DIST
+
+#################################
+# RECORDING #
+#################################
+
+record initial (initLaneSec.polygon.exterior.coords) as initLaneCoords
+record initial (leftLaneSec.polygon.exterior.coords) as leftLaneCoords
+record (ego.lane is initLaneSec.lane) as egoIsInInitLane
+record (ego.lane is leftLaneSec.lane) as egoIsInLeftLane
diff --git a/examples/dynamic_rulebook/multi_01/multi_01.sgraph b/examples/dynamic_rulebook/multi_01/multi_01.sgraph
new file mode 100644
index 0000000..c4217a3
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_01/multi_01.sgraph
@@ -0,0 +1,5 @@
+# ID 0
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+# Edge list
diff --git a/examples/dynamic_rulebook/multi_01/multi_01_00.graph b/examples/dynamic_rulebook/multi_01/multi_01_00.graph
new file mode 100644
index 0000000..40db04d
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_01/multi_01_00.graph
@@ -0,0 +1,6 @@
+# ID 0
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+# Edge list
+1 0
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_01/multi_01_01.graph b/examples/dynamic_rulebook/multi_01/multi_01_01.graph
new file mode 100644
index 0000000..03464c9
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_01/multi_01_01.graph
@@ -0,0 +1,6 @@
+# ID 1
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+# Edge list
+0 1
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_01/multi_01_02.graph b/examples/dynamic_rulebook/multi_01/multi_01_02.graph
new file mode 100644
index 0000000..50430a2
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_01/multi_01_02.graph
@@ -0,0 +1,5 @@
+# ID 2
+# Node list
+0 off rule0 monitor
+1 on rule1 monitor
+# Edge list
diff --git a/examples/dynamic_rulebook/multi_01/multi_01_rulebook.py b/examples/dynamic_rulebook/multi_01/multi_01_rulebook.py
new file mode 100644
index 0000000..f7e7a2f
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_01/multi_01_rulebook.py
@@ -0,0 +1,53 @@
+import numpy as np
+
+from verifai.rulebook import rulebook
+
+class rulebook_multi01(rulebook):
+ iteration = 0
+
+ def __init__(self, graph_path, rule_file, save_path=None, single_graph=False, using_sampler=-1, exploration_ratio=2.0):
+ rulebook.using_sampler = using_sampler
+ rulebook.exploration_ratio = exploration_ratio
+ super().__init__(graph_path, rule_file, single_graph=single_graph)
+ self.save_path = save_path
+
+ def evaluate(self, traj):
+ # Extract trajectory information
+ positions = np.array(traj.result.trajectory)
+ init_lane_coords = np.array(traj.result.records["initLaneCoords"])
+ left_lane_coords = np.array(traj.result.records["leftLaneCoords"])
+ ego_is_in_init_lane = np.array(traj.result.records["egoIsInInitLane"])
+ ego_is_in_left_lane = np.array(traj.result.records["egoIsInLeftLane"])
+
+ # Find switching points
+ switch_idx_1 = len(traj.result.trajectory)
+ switch_idx_2 = len(traj.result.trajectory)
+ distances_to_obs = positions[:, 0, :] - positions[:, 1, :]
+ distances_to_obs = np.linalg.norm(distances_to_obs, axis=1)
+ for i in range(len(distances_to_obs)):
+ if distances_to_obs[i] < 8.5 and switch_idx_1 == len(traj.result.trajectory):
+ switch_idx_1 = i
+ continue
+ if distances_to_obs[i] > 10 and switch_idx_1 < len(traj.result.trajectory) and switch_idx_2 == len(traj.result.trajectory):
+ switch_idx_2 = i
+ break
+ assert switch_idx_1 < len(traj.result.trajectory), "Switching point 1 cannot be found"
+
+ # Evaluation
+ indices_0 = np.arange(0, switch_idx_1)
+ indices_1 = np.arange(switch_idx_1, switch_idx_2)
+ indices_2 = np.arange(switch_idx_2, len(traj.result.trajectory))
+ if self.single_graph:
+ rho0 = self.evaluate_segment(traj, 0, indices_0)
+ rho1 = self.evaluate_segment(traj, 0, indices_1)
+ rho2 = self.evaluate_segment(traj, 0, indices_2)
+ print('Actual rho:')
+ print(rho0[0], rho0[1])
+ print(rho1[0], rho1[1])
+ print(rho2[0], rho2[1])
+ rho = self.evaluate_segment(traj, 0, np.arange(0, len(traj.result.trajectory)))
+ return np.array([rho])
+ rho0 = self.evaluate_segment(traj, 0, indices_0)
+ rho1 = self.evaluate_segment(traj, 1, indices_1)
+ rho2 = self.evaluate_segment(traj, 2, indices_2)
+ return np.array([rho0, rho1, rho2])
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_01/multi_01_spec.py b/examples/dynamic_rulebook/multi_01/multi_01_spec.py
new file mode 100644
index 0000000..5e6d093
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_01/multi_01_spec.py
@@ -0,0 +1,19 @@
+import numpy as np
+
+def rule0(simulation, indices): # safe distance to obstacle
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv1 = positions[indices, [0], :] - positions[indices, [1], :]
+ distances_to_adv1 = np.linalg.norm(distances_to_adv1, axis=1)
+ rho = np.min(distances_to_adv1, axis=0) - 3
+ return rho
+
+def rule1(simulation, indices): # ego is in the left lane
+ if indices.size == 0:
+ return 1
+ ego_is_in_left_lane = np.array(simulation.result.records["egoIsInLeftLane"], dtype=bool)
+ for i in indices:
+ if ego_is_in_left_lane[i][1]:
+ return -1
+ return 1
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_01/util/multi_01_analyze_diversity.py b/examples/dynamic_rulebook/multi_01/util/multi_01_analyze_diversity.py
new file mode 100644
index 0000000..9df31d8
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_01/util/multi_01_analyze_diversity.py
@@ -0,0 +1,69 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+directory = sys.argv[1]
+all_files = os.listdir(directory)
+all_files = [f for f in all_files if f.endswith('.csv') and f.startswith(sys.argv[2]+'.')]
+mode = sys.argv[3] # multi / single
+
+fig = plt.figure()
+ax = fig.add_subplot(projection='3d')
+count = 0
+ego_speed = []
+dist_threshold = []
+blocking_car_dist = []
+bypass_dist = []
+
+ego_speed_max = []
+dist_threshold_max = []
+blocking_car_dist_max = []
+bypass_dist_max = []
+
+for file in all_files:
+ infile = open(directory+'/'+file, 'r')
+ lines = infile.readlines()
+ if mode == 'single':
+ for i in range(1, len(lines)):
+ line = lines[i]
+ if float(line.split(',')[-1]) < 0 or float(line.split(',')[-2]) < 0:
+ ego_speed.append(float(line.split(',')[-3]))
+ dist_threshold.append(float(line.split(',')[-4]))
+ bypass_dist.append(float(line.split(',')[-5]))
+ blocking_car_dist.append(float(line.split(',')[-6]))
+ else:
+ for i in range(1, len(lines), 3):
+ line1 = lines[i]
+ line2 = lines[i+1]
+ line3 = lines[i+2]
+ if float(line2.split(',')[-1]) < 0 and float(line2.split(',')[-2]) < 0:
+ ego_speed_max.append(float(line1.split(',')[-3]))
+ dist_threshold_max.append(float(line1.split(',')[-4]))
+ bypass_dist_max.append(float(line1.split(',')[-5]))
+ blocking_car_dist_max.append(float(line1.split(',')[-6]))
+ else:
+ ego_speed.append(float(line1.split(',')[-3]))
+ dist_threshold.append(float(line1.split(',')[-4]))
+ bypass_dist.append(float(line1.split(',')[-5]))
+ blocking_car_dist.append(float(line1.split(',')[-6]))
+ #if float(line1.split(',')[-1]) < 0 or float(line1.split(',')[-2]) < 0 or float(line2.split(',')[-1]) < 0 or float(line2.split(',')[-2]) < 0 or float(line3.split(',')[-2]) < 0:
+ # ego_speed.append(float(line1.split(',')[-3]))
+ # dist_threshold.append(float(line1.split(',')[-4]))
+ # bypass_dist.append(float(line1.split(',')[-5]))
+ # blocking_car_dist.append(float(line1.split(',')[-6]))
+ #else:
+ # print(file, i)
+
+ax.scatter(ego_speed, dist_threshold, bypass_dist, c='b')
+ax.scatter(ego_speed_max, dist_threshold_max, bypass_dist_max, c='r')
+ax.set_xlabel('EGO_SPEED')
+ax.set_ylabel('DIST_THRESHOLD')
+ax.set_zlabel('BYPASS_DIST')
+plt.savefig(directory+'/'+sys.argv[2]+'_scatter.png')
+
+print("Standard deviation of ego_speed:", np.std(ego_speed), len(ego_speed))
+print("Standard deviation of dist_threshold:", np.std(dist_threshold), len(dist_threshold))
+print("Standard deviation of bypass_dist:", np.std(bypass_dist), len(bypass_dist))
+print("Standard deviation of blocking_car_dist:", np.std(blocking_car_dist), len(blocking_car_dist))
+print()
diff --git a/examples/dynamic_rulebook/multi_01/util/multi_01_collect_result.py b/examples/dynamic_rulebook/multi_01/util/multi_01_collect_result.py
new file mode 100644
index 0000000..074a3b2
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_01/util/multi_01_collect_result.py
@@ -0,0 +1,144 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import itertools
+
+infile = open(sys.argv[1], 'r') # *.txt
+mode = sys.argv[2] # multi / single
+order = sys.argv[3]
+
+# error weights
+result_count_0 = [[] for i in range(3)]
+result_count_1 = [[] for i in range(3)]
+result_count_2 = [[] for i in range(3)]
+# counterexample types
+counterexample_type_0 = [{} for i in range(3)]
+counterexample_type_1 = [{} for i in range(3)]
+counterexample_type_2 = [{} for i in range(3)]
+curr_source = 0
+lines = infile.readlines()
+infile.close()
+
+for i in range(len(lines)):
+ if mode == 'multi':
+ if 'RHO' in lines[i]:
+ line = lines[i+1].strip().split(' ')
+ val1 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val1.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val1) == 2, 'Invalid length of rho'
+ result_count_0[curr_source].append(val1[0]*1 + val1[1]*2)
+ if tuple(1*np.array([val1[0], val1[1]])) in counterexample_type_0[curr_source]:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[1]]))] += 1
+ else:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[1]]))] = 1
+
+ line = lines[i+2].strip().split(' ')
+ val2 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val2.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val2) == 2, 'Invalid length of rho'
+ result_count_1[curr_source].append(val2[0]*2 + val2[1]*1)
+ if tuple(1*np.array([val2[0], val2[1]])) in counterexample_type_1[curr_source]:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1]]))] += 1
+ else:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1]]))] = 1
+
+ line = lines[i+3].strip().split(' ')
+ val3 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val3.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val3) == 2, 'Invalid length of rho'
+ result_count_2[curr_source].append(val3[1]*1)
+ if tuple(1*np.array([val3[1]])) in counterexample_type_2[curr_source]:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[1]]))] += 1
+ else:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[1]]))] = 1
+
+ if order == '-1':
+ curr_source = curr_source + 1 if curr_source < 2 else 0
+ else:
+ if 'Actual rho' in lines[i]:
+ line = lines[i+1].strip().split(' ')
+ val1 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val1.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val1) == 2, 'Invalid length of rho'
+ result_count_0[curr_source].append(val1[0]*1 + val1[1]*2)
+ if tuple(1*np.array([val1[0], val1[1]])) in counterexample_type_0[curr_source]:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[1]]))] += 1
+ else:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[1]]))] = 1
+
+ line = lines[i+2].strip().split(' ')
+ val2 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val2.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val2) == 2, 'Invalid length of rho'
+ result_count_1[curr_source].append(val2[0]*2 + val2[1]*1)
+ if tuple(1*np.array([val2[0], val2[1]])) in counterexample_type_1[curr_source]:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1]]))] += 1
+ else:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1]]))] = 1
+
+ line = lines[i+3].strip().split(' ')
+ val3 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val3.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val3) == 2, 'Invalid length of rho'
+ result_count_2[curr_source].append(val3[1]*1)
+ if tuple(1*np.array([val3[1]])) in counterexample_type_2[curr_source]:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[1]]))] += 1
+ else:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[1]]))] = 1
+
+ if order == '-1':
+ curr_source = curr_source + 1 if curr_source < 2 else 0
+
+print('Error weights')
+print('segment 0:')
+for i in range(1):
+ print('average:', np.mean(result_count_0[i]), 'max:', np.max(result_count_0[i]), 'percentage:', float(np.count_nonzero(result_count_0[i])/len(result_count_0[i])), result_count_0[i])
+print('segment 1:')
+for i in range(1):
+ print('average:', np.mean(result_count_1[i]), 'max:', np.max(result_count_1[i]), 'percentage:', float(np.count_nonzero(result_count_1[i])/len(result_count_1[i])), result_count_1[i])
+print('segment 2:')
+for i in range(1):
+ print('average:', np.mean(result_count_2[i]), 'max:', np.max(result_count_2[i]), 'percentage:', float(np.count_nonzero(result_count_2[i])/len(result_count_2[i])), result_count_2[i])
+
+print('\nCounterexample types')
+print('segment 0:')
+for i in range(1):
+ print('Types:', len(counterexample_type_0[i]))
+ for key, value in reversed(sorted(counterexample_type_0[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print('segment 1:')
+for i in range(1):
+ print('Types:', len(counterexample_type_1[i]))
+ for key, value in reversed(sorted(counterexample_type_1[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print('segment 2:')
+for i in range(1):
+ print('Types:', len(counterexample_type_2[i]))
+ for key, value in reversed(sorted(counterexample_type_2[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print()
diff --git a/examples/dynamic_rulebook/multi_02/multi_02.py b/examples/dynamic_rulebook/multi_02/multi_02.py
new file mode 100644
index 0000000..44e8e97
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_02/multi_02.py
@@ -0,0 +1,52 @@
+import sys
+import os
+sys.path.append(os.path.abspath("."))
+import random
+import numpy as np
+
+from multi import *
+from multi_02_rulebook import rulebook_multi02
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--scenic-path', '-sp', type=str, default='uberCrashNewton.scenic',
+ help='Path to Scenic script')
+ parser.add_argument('--graph-path', '-gp', type=str, default=None,
+ help='Path to graph file')
+ parser.add_argument('--rule-path', '-rp', type=str, default=None,
+ help='Path to rule file')
+ parser.add_argument('--output-dir', '-o', type=str, default=None,
+ help='Directory to save output trajectories')
+ parser.add_argument('--output-csv-dir', '-co', type=str, default=None,
+ help='Directory to save output error tables (csv files)')
+ parser.add_argument('--parallel', action='store_true')
+ parser.add_argument('--num-workers', type=int, default=5, help='Number of parallel workers')
+ parser.add_argument('--sampler-type', '-s', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--experiment-name', '-e', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--model', '-m', type=str, default='scenic.simulators.newtonian.driving_model')
+ parser.add_argument('--headless', action='store_true')
+ parser.add_argument('--n-iters', '-n', type=int, default=None, help='Number of simulations to run')
+ parser.add_argument('--max-time', type=int, default=None, help='Maximum amount of time to run simulations')
+ parser.add_argument('--single-graph', action='store_true', help='Only a unified priority graph')
+ parser.add_argument('--seed', type=int, default=0, help='Random seed')
+ parser.add_argument('--using-sampler', type=int, default=-1, help='Assigning sampler to use')
+ parser.add_argument('--exploration-ratio', type=float, default=2.0, help='Exploration ratio')
+ parser.add_argument('--use-dependency', action='store_true', help='Use dependency')
+ parser.add_argument('--using-continuous', action='store_true', help='Using continuous')
+ args = parser.parse_args()
+ if args.n_iters is None and args.max_time is None:
+ raise ValueError('At least one of --n-iters or --max-time must be set')
+
+ random.seed(args.seed)
+ np.random.seed(args.seed)
+
+ rb = rulebook_multi02(args.graph_path, args.rule_path, save_path=args.output_dir, single_graph=args.single_graph, using_sampler=args.using_sampler,
+ exploration_ratio=args.exploration_ratio, use_dependency=args.use_dependency, using_continuous=args.using_continuous)
+ run_experiments(args.scenic_path, rulebook=rb,
+ parallel=args.parallel, model=args.model,
+ sampler_type=args.sampler_type, headless=args.headless,
+ num_workers=args.num_workers, output_dir=args.output_csv_dir, experiment_name=args.experiment_name,
+ max_time=args.max_time, n_iters=args.n_iters)
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_02/multi_02.scenic b/examples/dynamic_rulebook/multi_02/multi_02.scenic
new file mode 100644
index 0000000..b32b9fd
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_02/multi_02.scenic
@@ -0,0 +1,129 @@
+"""
+TITLE: Multi 02
+AUTHOR: Kai-Chun Chang, kaichunchang@berkeley.edu
+"""
+
+#################################
+# MAP AND MODEL #
+#################################
+
+param map = localPath('../maps/Town05.xodr')
+param carla_map = 'Town05'
+param N = 4
+model scenic.domains.driving.model
+#model scenic.simulators.carla.model
+
+#################################
+# CONSTANTS #
+#################################
+
+MODEL = 'vehicle.lincoln.mkz_2017'
+
+param EGO_SPEED = VerifaiRange(8, 12)
+param EGO_BRAKE = VerifaiRange(0.7, 1.0)
+param ADV_SPEED = VerifaiRange(3, 6)
+param ADV3_SPEED = VerifaiRange(3, 6) #VerifaiRange(1, 3)
+
+ADV1_DIST = 12
+ADV2_DIST = -6
+ADV3_DIST = 6 #18
+
+BYPASS_DIST = 10
+SAFE_DIST = 10
+INIT_DIST = 40
+TERM_DIST = 80
+
+#################################
+# AGENT BEHAVIORS #
+#################################
+
+behavior DecelerateBehavior(brake):
+ take SetBrakeAction(brake)
+
+behavior EgoBehavior():
+ try:
+ do FollowLaneBehavior(target_speed=globalParameters.EGO_SPEED)
+ interrupt when (distance from adv2 to ego) > BYPASS_DIST:
+ fasterLaneSec = self.laneSection.fasterLane
+ do LaneChangeBehavior(
+ laneSectionToSwitch=fasterLaneSec,
+ target_speed=globalParameters.EGO_SPEED)
+ try:
+ do FollowLaneBehavior(
+ target_speed=globalParameters.EGO_SPEED,
+ laneToFollow=fasterLaneSec.lane)
+ interrupt when (distance from adv3 to ego) < SAFE_DIST:
+ do DecelerateBehavior(brake=globalParameters.EGO_BRAKE)
+ interrupt when (distance from adv1 to ego) < SAFE_DIST:
+ do DecelerateBehavior(brake=globalParameters.EGO_BRAKE)
+
+behavior Adv1Behavior():
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+
+behavior Adv2Behavior():
+ fasterLaneSec = self.laneSection.fasterLane
+ do LaneChangeBehavior(
+ laneSectionToSwitch=fasterLaneSec,
+ target_speed=globalParameters.ADV_SPEED)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+
+behavior Adv3Behavior():
+ fasterLaneSec = self.laneSection.fasterLane
+ do LaneChangeBehavior(
+ laneSectionToSwitch=fasterLaneSec,
+ target_speed=globalParameters.ADV_SPEED)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV3_SPEED)
+
+#################################
+# SPATIAL RELATIONS #
+#################################
+
+initLane = Uniform(*network.lanes)
+egoSpawnPt = new OrientedPoint in initLane.centerline
+
+#################################
+# SCENARIO SPECIFICATION #
+#################################
+
+ego = new Car at egoSpawnPt,
+ with blueprint MODEL,
+ with behavior EgoBehavior()
+
+adv1 = new Car following roadDirection for ADV1_DIST,
+ with blueprint MODEL,
+ with behavior Adv1Behavior()
+
+adv2 = new Car following roadDirection for ADV2_DIST,
+ with blueprint MODEL,
+ with behavior Adv2Behavior()
+
+adv3 = new Car following roadDirection for ADV3_DIST,
+ with blueprint MODEL,
+ with behavior Adv3Behavior()
+
+require (distance to intersection) > INIT_DIST
+require (distance from adv1 to intersection) > INIT_DIST
+require (distance from adv2 to intersection) > INIT_DIST
+require (distance from adv3 to intersection) > INIT_DIST
+require always (adv1.laneSection._fasterLane is not None)
+terminate when (distance to egoSpawnPt) > TERM_DIST
+
+#################################
+# RECORDING #
+#################################
+
+#record initial (adv2.lane.polygon.exterior.coords) as egoStartLaneCoords
+#record final (adv2.lane.polygon.exterior.coords) as egoEndLaneCoords
+record (ego.lane is initLane or ego.lane is not adv2.lane) as egoIsInInitLane
+record (adv2.lane is initLane) as adv2IsInInitLane # start evaluation only when adv2 reaches another lane
+record (adv3.lane is initLane) as adv3IsInInitLane # start evaluation only when adv3 reaches another lane
+
+record ego._boundingPolygon as egoPoly
+record adv1._boundingPolygon as adv1Poly
+record adv2._boundingPolygon as adv2Poly
+record adv3._boundingPolygon as adv3Poly
+
+record ego.laneSection.polygon as egoLanePoly
+record adv1.laneSection.polygon as adv1LanePoly
+record adv2.laneSection.polygon as adv2LanePoly
+record adv3.laneSection.polygon as adv3LanePoly
diff --git a/examples/dynamic_rulebook/multi_02/multi_02.sgraph b/examples/dynamic_rulebook/multi_02/multi_02.sgraph
new file mode 100644
index 0000000..8a113e8
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_02/multi_02.sgraph
@@ -0,0 +1,9 @@
+# ID 0
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+2 on rule2 monitor
+3 on rule3 monitor
+# Edge list
+0 1
+3 2
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_02/multi_02_00.graph b/examples/dynamic_rulebook/multi_02/multi_02_00.graph
new file mode 100644
index 0000000..ad819da
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_02/multi_02_00.graph
@@ -0,0 +1,8 @@
+# ID 0
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+2 off rule2 monitor
+3 off rule3 monitor
+# Edge list
+0 1
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_02/multi_02_01.graph b/examples/dynamic_rulebook/multi_02/multi_02_01.graph
new file mode 100644
index 0000000..ef0b8d5
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_02/multi_02_01.graph
@@ -0,0 +1,8 @@
+# ID 1
+# Node list
+0 off rule0 monitor
+1 off rule1 monitor
+2 on rule2 monitor
+3 on rule3 monitor
+# Edge list
+3 2
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_02/multi_02_rulebook.py b/examples/dynamic_rulebook/multi_02/multi_02_rulebook.py
new file mode 100644
index 0000000..fd27a79
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_02/multi_02_rulebook.py
@@ -0,0 +1,68 @@
+import numpy as np
+
+from verifai.rulebook import rulebook
+
+class rulebook_multi02(rulebook):
+ iteration = 0
+
+ def __init__(self, graph_path, rule_file, save_path=None, single_graph=False, using_sampler=-1, exploration_ratio=2.0, use_dependency=False, using_continuous=False):
+ rulebook.using_sampler = using_sampler
+ rulebook.exploration_ratio = exploration_ratio
+ rulebook.using_continuous = using_continuous
+ super().__init__(graph_path, rule_file, single_graph=single_graph)
+ self.save_path = save_path
+ self.use_dependency = use_dependency
+
+ def evaluate(self, traj):
+ # Extract trajectory information
+ positions = np.array(traj.result.trajectory)
+ #ego_start_lane_coords = np.array(traj.result.records["egoStartLaneCoords"])
+ #ego_end_lane_coords = np.array(traj.result.records["egoEndLaneCoords"])
+ ego_is_in_init_lane = np.array(traj.result.records["egoIsInInitLane"])
+ adv2_is_in_init_lane = np.array(traj.result.records["adv2IsInInitLane"])
+ adv3_is_in_init_lane = np.array(traj.result.records["adv3IsInInitLane"])
+
+ # Find starting point, i.e., adv2 and adv3 have reached the new lane
+ start_idx = -1
+ for i in range(len(adv2_is_in_init_lane)):
+ if adv2_is_in_init_lane[i][1] == 0 and adv3_is_in_init_lane[i][1] == 0:
+ start_idx = i
+ break
+ assert start_idx != -1, "Starting point not found"
+
+ # Find switching point, i.e., ego has reached the new lane
+ switch_idx = len(traj.result.trajectory)
+ for i in range(start_idx, len(ego_is_in_init_lane)):
+ if ego_is_in_init_lane[i][1] == 0:
+ switch_idx = i
+ break
+ assert switch_idx > start_idx, "Switching point should be larger than starting point"
+
+ # Evaluation
+ indices_0 = np.arange(start_idx, switch_idx)
+ indices_1 = np.arange(switch_idx, len(traj.result.trajectory))
+ if self.single_graph:
+ rho0 = self.evaluate_segment(traj, 0, indices_0)
+ rho1 = self.evaluate_segment(traj, 0, indices_1)
+ print('Actual rho:', rho0, rho1)
+ rho = self.evaluate_segment(traj, 0, np.arange(0, len(traj.result.trajectory)))
+ return np.array([rho])
+ rho0 = self.evaluate_segment(traj, 0, indices_0)
+ rho1 = self.evaluate_segment(traj, 1, indices_1)
+ if rulebook.using_continuous:
+ print('Original rho:', rho0[0], rho0[1], rho1[2], rho1[3])
+ print('Normalized rho:', rho0[0]/2.0, rho0[1]/2.5, rho1[2]/8.0, rho1[3]/8.0)
+ rho0[0] = rho0[0]/2.0
+ rho0[1] = rho0[1]/2.5
+ rho1[2] = rho1[2]/8.0
+ rho1[3] = rho1[3]/8.0
+ if self.use_dependency:
+ print('Before dependency weighting:', rho0[0], rho0[1], rho1[2], rho1[3])
+ rho01 = rho0[1] - 0.879 * rho1[2]
+ rho12 = rho1[2] - 0.879 * rho0[1]
+ print('After dependency weighting:', rho0[0], rho01, rho12, rho1[3])
+ print('rho01 toggles:', np.sign(rho01) != np.sign(rho0[1]))
+ print('rho12 toggles:', np.sign(rho12) != np.sign(rho1[2]))
+ rho0[1] = rho01
+ rho1[2] = rho12
+ return np.array([rho0, rho1])
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_02/multi_02_spec.py b/examples/dynamic_rulebook/multi_02/multi_02_spec.py
new file mode 100644
index 0000000..573d1c3
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_02/multi_02_spec.py
@@ -0,0 +1,41 @@
+import numpy as np
+
+def rule0(simulation, indices): # safe distance to adv1
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv1 = positions[indices, [0], :] - positions[indices, [1], :]
+ distances_to_adv1 = np.linalg.norm(distances_to_adv1, axis=1)
+ rho = np.min(distances_to_adv1, axis=0) - 8
+ return rho
+
+def rule1(simulation, indices): # reach overtaking distance to adv2
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv2 = positions[indices, [0], :] - positions[indices, [2], :]
+ distances_to_adv2 = np.linalg.norm(distances_to_adv2, axis=1)
+ rho = np.max(distances_to_adv2, axis=0) - 10
+ if rho < 0:
+ return rho
+ elif np.max(indices) == len(simulation.result.trajectory) - 1: # lane change is not actually completed
+ return -0.1
+ return rho
+
+def rule2(simulation, indices): # safe distance to adv2 after lane change
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv2 = positions[indices, [0], :] - positions[indices, [2], :]
+ distances_to_adv2 = np.linalg.norm(distances_to_adv2, axis=1)
+ rho = np.min(distances_to_adv2, axis=0) - 8
+ return rho
+
+def rule3(simulation, indices): # safe distance to adv3
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv3 = positions[indices, [0], :] - positions[indices, [3], :]
+ distances_to_adv3 = np.linalg.norm(distances_to_adv3, axis=1)
+ rho = np.min(distances_to_adv3, axis=0) - 8
+ return rho
diff --git a/examples/dynamic_rulebook/multi_02/util/multi_02_analyze_diversity.py b/examples/dynamic_rulebook/multi_02/util/multi_02_analyze_diversity.py
new file mode 100644
index 0000000..2405228
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_02/util/multi_02_analyze_diversity.py
@@ -0,0 +1,61 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+directory = sys.argv[1]
+all_files = os.listdir(directory)
+all_files = [f for f in all_files if f.endswith('.csv') and f.startswith(sys.argv[2]+'.')]
+mode = sys.argv[3] # multi / single
+
+fig = plt.figure()
+ax = fig.add_subplot(projection='3d')
+count = 0
+adv3_speed = []
+adv_speed = []
+ego_brake = []
+ego_speed = []
+adv3_speed_seg0_max = []
+adv_speed_seg0_max = []
+ego_brake_seg0_max = []
+ego_speed_seg0_max = []
+for file in all_files:
+ infile = open(directory+'/'+file, 'r')
+ lines = infile.readlines()
+ if mode == 'single':
+ for i in range(1, len(lines)):
+ line = lines[i]
+ if float(line.split(',')[-1]) < 0 or float(line.split(',')[-2]) < 0 or float(line.split(',')[-3]) < 0 or float(line.split(',')[-4]) < 0:
+ ego_speed.append(float(line.split(',')[-5]))
+ ego_brake.append(float(line.split(',')[-6]))
+ adv_speed.append(float(line.split(',')[-7]))
+ adv3_speed.append(float(line.split(',')[-8]))
+ else:
+ for i in range(1, len(lines), 2):
+ line1 = lines[i]
+ line2 = lines[i+1]
+ if float(line1.split(',')[-3]) < 0 and float(line1.split(',')[-2]) < 0:
+ ego_speed_seg0_max.append(float(line1.split(',')[-5]))
+ ego_brake_seg0_max.append(float(line1.split(',')[-6]))
+ adv_speed_seg0_max.append(float(line1.split(',')[-7]))
+ adv3_speed_seg0_max.append(float(line1.split(',')[-8]))
+ elif float(line1.split(',')[-3]) < 0 or float(line1.split(',')[-2]) < 0 or float(line2.split(',')[-1]) < 0 or float(line2.split(',')[-4]) < 0:
+ ego_speed.append(float(line1.split(',')[-5]))
+ ego_brake.append(float(line1.split(',')[-6]))
+ adv_speed.append(float(line1.split(',')[-7]))
+ adv3_speed.append(float(line1.split(',')[-8]))
+ else:
+ print(file, i)
+
+ax.scatter(ego_speed_seg0_max, ego_brake_seg0_max, adv_speed_seg0_max, c='r')
+ax.scatter(ego_speed, ego_brake, adv_speed, c='b')
+ax.set_xlabel('EGO_SPEED')
+ax.set_ylabel('EGO_BRAKE')
+ax.set_zlabel('ADV_SPEED')
+plt.savefig(directory+'/'+sys.argv[2]+'_scatter.png')
+
+print("Standard deviation of ego_speed:", np.std(ego_speed), len(ego_speed))
+print("Standard deviation of ego_brake:", np.std(ego_brake), len(ego_brake))
+print("Standard deviation of adv_speed:", np.std(adv_speed), len(adv_speed))
+print("Standard deviation of adv3_speed:", np.std(adv3_speed), len(adv3_speed))
+print()
diff --git a/examples/dynamic_rulebook/multi_02/util/multi_02_collect_result.py b/examples/dynamic_rulebook/multi_02/util/multi_02_collect_result.py
new file mode 100644
index 0000000..cc80488
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_02/util/multi_02_collect_result.py
@@ -0,0 +1,127 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+
+infile = open(sys.argv[1], 'r') # *.txt
+mode = sys.argv[2] # multi / single
+order = sys.argv[3] # -1 / 0 / 1
+
+# error weights
+result_count_0 = [[] for i in range(3)]
+result_count_1 = [[] for i in range(3)]
+# counterexample types
+counterexample_type_0 = [{} for i in range(3)]
+counterexample_type_1 = [{} for i in range(3)]
+#result_count_0 = np.zeros(shape=(2,4), dtype=int) # result_count_0[i] = [count of 00, 01, 10, 11 in segment 0] sampled from sampler i
+#result_count_1 = np.zeros(shape=(2,4), dtype=int) # result_count_1[i] = [count of 00, 01, 10, 11 in segment 1] sampled from sampler i
+curr_source = 0
+lines = infile.readlines()
+infile.close()
+
+count = 0
+
+for i in range(len(lines)):
+ if order == '0':
+ curr_source = 0
+ elif order == '1':
+ curr_source = 1
+ if mode == 'multi':
+ if 'Rho' in lines[i]:
+ line = lines[i].strip()
+ seg1 = line[line.find('[[')+2:line.find(']')].split(' ')
+ val1 = []
+ for s in seg1:
+ if s != '':
+ val1.append(float(s) < 0)
+ assert len(val1) == 4, 'Invalid length of rho'
+ result_count_0[curr_source].append(val1[0]*2 + val1[1]*1)
+ if tuple(1*np.array([val1[0], val1[1]])) in counterexample_type_0[curr_source]:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[1]]))] += 1
+ else:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[1]]))] = 1
+ #result_count_0[curr_source][val1[0]*2 + val1[1]*1] += 1
+
+ line = lines[i+1].strip()
+ seg2 = line[line.find('[')+1:line.find(']]')].split(' ')
+ val2 = []
+ for s in seg2:
+ if s != '':
+ val2.append(float(s) < 0)
+ assert len(val2) == 4, 'Invalid length of rho'
+ result_count_1[curr_source].append(val2[3]*2 + val2[2]*1)
+ if tuple(1*np.array([val2[3], val2[2]])) in counterexample_type_1[curr_source]:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[3], val2[2]]))] += 1
+ else:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[3], val2[2]]))] = 1
+ #result_count_1[curr_source][val2[3]*2 + val2[2]*1] += 1
+
+ if order == '-1':
+ curr_source = 1 - curr_source
+
+ count += 1
+ if count == 900:
+ break
+ else:
+ if 'Actual rho' in lines[i]:
+ line = lines[i].strip()
+ seg1 = line[line.find('[')+1:line.find(']')].split(' ')
+ val1 = []
+ for s in seg1:
+ if s != '':
+ val1.append(float(s) < 0)
+ assert len(val1) == 4, 'Invalid length of rho'
+ result_count_0[curr_source].append(val1[0]*2 + val1[1]*1)
+ if tuple(1*np.array([val1[0], val1[1]])) in counterexample_type_0[curr_source]:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[1]]))] += 1
+ else:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[1]]))] = 1
+ #result_count_0[curr_source][val1[0]*2 + val1[1]*1] += 1
+
+ seg2 = line[line.find('] [')+3:-1].split(' ')
+ val2 = []
+ for s in seg2:
+ if s != '':
+ val2.append(float(s) < 0)
+ assert len(val2) == 4, 'Invalid length of rho'
+ result_count_1[curr_source].append(val2[3]*2 + val2[2]*1)
+ if tuple(1*np.array([val2[3], val2[2]])) in counterexample_type_1[curr_source]:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[3], val2[2]]))] += 1
+ else:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[3], val2[2]]))] = 1
+ #result_count_1[curr_source][val2[3]*2 + val2[2]*1] += 1
+
+print('Error weights')
+print('segment 0:')
+for i in range(1):
+ print('average:', np.mean(result_count_0[i]), 'max:', np.max(result_count_0[i]), 'percentage:', float(np.count_nonzero(result_count_0[i])/len(result_count_0[i])), result_count_0[i])
+print('segment 1:')
+for i in range(1):
+ print('average:', np.mean(result_count_1[i]), 'max:', np.max(result_count_1[i]), 'percentage:', float(np.count_nonzero(result_count_1[i])/len(result_count_1[i])), result_count_1[i])
+
+print('\nCounterexample types')
+print('segment 0:')
+for i in range(1):
+ print('Types:', len(counterexample_type_0[i]))
+ for key, value in reversed(sorted(counterexample_type_0[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print('segment 1:')
+for i in range(1):
+ print('Types:', len(counterexample_type_1[i]))
+ for key, value in reversed(sorted(counterexample_type_1[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print()
+
+#rows = ['from sampler 0', 'from sampler 1']
+##cols = ['(r0, r1) = 00', '(r0, r1) = 01', '(r0, r1) = 10', '(r0, r1) = 11']
+#print('Falsification result in segment 0:')
+#print(result_count_0[0][0], result_count_0[0][1], result_count_0[0][2], result_count_0[0][3])
+#print(result_count_0[1][0], result_count_0[1][1], result_count_0[1][2], result_count_0[1][3])
+##df = pd.DataFrame(result_count_0, columns=cols, index=rows)
+##print('Falsification result in segment 0:\n', df, '\n')
+##cols = ['(r3, r2) = 00', '(r3, r2) = 01', '(r3, r2) = 10', '(r3, r2) = 11']
+#print('Falsification result in segment 1:')
+#print(result_count_1[0][0], result_count_1[0][1], result_count_1[0][2], result_count_1[0][3])
+#print(result_count_1[1][0], result_count_1[1][1], result_count_1[1][2], result_count_1[1][3])
+##df = pd.DataFrame(result_count_1, columns=cols, index=rows)
+##print('Falsification result in segment 1:\n', df)
diff --git a/examples/dynamic_rulebook/multi_03/multi_03.py b/examples/dynamic_rulebook/multi_03/multi_03.py
new file mode 100644
index 0000000..5a586e3
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_03/multi_03.py
@@ -0,0 +1,51 @@
+import sys
+import os
+sys.path.append(os.path.abspath("."))
+import random
+import numpy as np
+
+from multi import *
+from multi_03_rulebook import rulebook_multi03
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--scenic-path', '-sp', type=str, default='uberCrashNewton.scenic',
+ help='Path to Scenic script')
+ parser.add_argument('--graph-path', '-gp', type=str, default=None,
+ help='Path to graph file')
+ parser.add_argument('--rule-path', '-rp', type=str, default=None,
+ help='Path to rule file')
+ parser.add_argument('--output-dir', '-o', type=str, default=None,
+ help='Directory to save output trajectories')
+ parser.add_argument('--output-csv-dir', '-co', type=str, default=None,
+ help='Directory to save output error tables (csv files)')
+ parser.add_argument('--parallel', action='store_true')
+ parser.add_argument('--num-workers', type=int, default=5, help='Number of parallel workers')
+ parser.add_argument('--sampler-type', '-s', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--experiment-name', '-e', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--model', '-m', type=str, default='scenic.simulators.newtonian.driving_model')
+ parser.add_argument('--headless', action='store_true')
+ parser.add_argument('--n-iters', '-n', type=int, default=None, help='Number of simulations to run')
+ parser.add_argument('--max-time', type=int, default=None, help='Maximum amount of time to run simulations')
+ parser.add_argument('--single-graph', action='store_true', help='Only a unified priority graph')
+ parser.add_argument('--seed', type=int, default=0, help='Random seed')
+ parser.add_argument('--using-sampler', type=int, default=-1, help='Assigning sampler to use')
+ parser.add_argument('--max-simulation-steps', type=int, default=300, help='Maximum number of simulation steps')
+ parser.add_argument('--exploration-ratio', type=float, default=2.0, help='Exploration ratio')
+ args = parser.parse_args()
+ if args.n_iters is None and args.max_time is None:
+ raise ValueError('At least one of --n-iters or --max-time must be set')
+
+ random.seed(args.seed)
+ np.random.seed(args.seed)
+
+ rb = rulebook_multi03(args.graph_path, args.rule_path, save_path=args.output_dir, single_graph=args.single_graph,
+ using_sampler=args.using_sampler, exploration_ratio=args.exploration_ratio)
+ run_experiments(args.scenic_path, rulebook=rb,
+ parallel=args.parallel, model=args.model,
+ sampler_type=args.sampler_type, headless=args.headless,
+ num_workers=args.num_workers, output_dir=args.output_csv_dir, experiment_name=args.experiment_name,
+ max_time=args.max_time, n_iters=args.n_iters, max_steps=args.max_simulation_steps)
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_03/multi_03.scenic b/examples/dynamic_rulebook/multi_03/multi_03.scenic
new file mode 100644
index 0000000..e7419bd
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_03/multi_03.scenic
@@ -0,0 +1,178 @@
+"""
+TITLE: Multi 03
+AUTHOR: Kai-Chun Chang, kaichunchang@berkeley.edu
+"""
+
+#################################
+# MAP AND MODEL #
+#################################
+
+param map = localPath('../maps/Town05.xodr')
+param carla_map = 'Town05'
+param N = 11
+model scenic.domains.driving.model
+
+#################################
+# CONSTANTS #
+#################################
+
+MODEL = 'vehicle.lincoln.mkz_2017' #'vehicle.toyota.prius'
+MODEL_ADV = 'vehicle.lincoln.mkz_2017'
+
+EGO_INIT_DIST = [30, 40]
+param EGO_SPEED = VerifaiRange(7, 10)
+EGO_BRAKE = 1.0
+
+ADV1_DIST = -8
+ADV_INIT_DIST = [15, 25]
+param ADV_SPEED = VerifaiRange(5, 8)
+param ADV1_SPEED = VerifaiRange(9, 12)
+param ADV2_SPEED = VerifaiRange(4, 7)
+ADV_BRAKE = 1.0
+
+PED_MIN_SPEED = 1.0
+PED_THRESHOLD = 20
+PED_FINAL_SPEED = 1.0
+
+#param SAFETY_DIST = VerifaiRange(8, 12)
+SAFETY_DIST = 8
+CRASH_DIST = 5
+TERM_DIST = 80
+
+#################################
+# AGENT BEHAVIORS #
+#################################
+
+behavior EgoBehavior(trajectory):
+ flag = True
+ try:
+ do FollowTrajectoryBehavior(target_speed=globalParameters.EGO_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+ interrupt when withinDistanceToAnyObjs(self, SAFETY_DIST) and (ped in network.drivableRegion) and flag:
+ flag = False
+ while withinDistanceToAnyObjs(self, SAFETY_DIST + 3):
+ take SetBrakeAction(EGO_BRAKE)
+
+behavior Adv1Behavior(trajectory):
+ try:
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV1_SPEED, trajectory=trajectory)
+ #do FollowLaneBehavior(target_speed=globalParameters.ADV1_SPEED)
+ interrupt when (distance from adv1 to ego) < SAFETY_DIST:
+ #interrupt when (distance from adv1 to ego) < SAFETY_DIST + 3:
+ take SetBrakeAction(ADV_BRAKE)
+
+behavior Adv2Behavior(trajectory):
+ try:
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV2_SPEED)
+ interrupt when (distance from self to ped) < SAFETY_DIST:
+ # take SetBrakeAction(ADV_BRAKE)
+ #interrupt when withinDistanceToAnyObjs(self, SAFETY_DIST + 3):
+ take SetBrakeAction(ADV_BRAKE)
+
+behavior Adv3Behavior(trajectory):
+ try:
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+ interrupt when (distance from self to ped) < SAFETY_DIST:
+ # take SetBrakeAction(ADV_BRAKE)
+ #interrupt when withinDistanceToAnyObjs(self, SAFETY_DIST + 3):
+ take SetBrakeAction(ADV_BRAKE)
+
+behavior Adv4Behavior(trajectory):
+ try:
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV_SPEED, trajectory=trajectory)
+ interrupt when withinDistanceToAnyObjs(self, SAFETY_DIST):
+ take SetBrakeAction(ADV_BRAKE)
+
+behavior Pedbehavior():
+ take SetWalkingSpeedAction(speed=PED_MIN_SPEED)
+
+#################################
+# SPATIAL RELATIONS #
+#################################
+
+intersection = Uniform(*filter(lambda i: i.is4Way, network.intersections))
+
+# ego: right turn from S to E
+egoManeuver = Uniform(*filter(lambda m: m.type is ManeuverType.RIGHT_TURN, intersection.maneuvers))
+egoInitLane = egoManeuver.startLane
+egoTrajectory = [egoInitLane, egoManeuver.connectingLane, egoManeuver.endLane]
+egoSpawnPt = new OrientedPoint in egoInitLane.centerline
+
+# adv1: straight from S to N
+adv1InitLane = egoInitLane
+adv1Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv1InitLane.maneuvers))
+adv1Trajectory = [adv1InitLane, adv1Maneuver.connectingLane, adv1Maneuver.endLane]
+
+# adv2: straight from W to E
+adv2InitLane = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT,
+ Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, egoInitLane.maneuvers)).conflictingManeuvers)).startLane
+adv2Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv2InitLane.maneuvers))
+adv2Trajectory = [adv2InitLane, adv2Maneuver.connectingLane, adv2Maneuver.endLane]
+adv2SpawnPt = new OrientedPoint in adv2InitLane.centerline
+
+# adv3: left-turn from E to S
+adv3InitLane = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv2Maneuver.reverseManeuvers)).startLane
+adv3Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.LEFT_TURN, adv3InitLane.maneuvers))
+adv3Trajectory = [adv3InitLane, adv3Maneuver.connectingLane, adv3Maneuver.endLane]
+
+# adv4: left-turn from N to E
+adv4InitLane = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT,
+ Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, egoInitLane.maneuvers)).reverseManeuvers)).startLane
+adv4Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.LEFT_TURN, adv4InitLane.maneuvers))
+adv4Trajectory = [adv4InitLane, adv4Maneuver.connectingLane, adv4Maneuver.endLane]
+
+# pedestrian
+tempSpawnPt = egoInitLane.centerline[-1]
+pedSpawnPt = new OrientedPoint right of tempSpawnPt by 5
+pedEndPt = new OrientedPoint at pedSpawnPt offset by (0, 5, 0)
+
+#################################
+# SCENARIO SPECIFICATION #
+#################################
+
+ego = new Car at egoSpawnPt,
+ with blueprint MODEL,
+ with behavior EgoBehavior(egoTrajectory)
+
+adv1 = new Car following roadDirection for ADV1_DIST,
+ with blueprint MODEL_ADV,
+ with behavior Adv1Behavior(adv1Trajectory)
+
+adv2 = new Car at adv2SpawnPt,
+ with blueprint MODEL_ADV,
+ with behavior Adv2Behavior(adv2Trajectory)
+
+adv3 = new Car at adv2 offset by -10 @ 70,
+ with blueprint MODEL_ADV,
+ with behavior Adv3Behavior(adv3Trajectory)
+
+adv4 = new Car at ego offset by -10 @ 85,
+ with blueprint MODEL_ADV,
+ with behavior Adv3Behavior(adv4Trajectory)
+
+ped = new Pedestrian at pedSpawnPt,
+ facing toward pedEndPt,
+ with regionContainedIn None,
+ with behavior Pedbehavior()
+
+require EGO_INIT_DIST[0] <= (distance to intersection) <= EGO_INIT_DIST[1]
+require ADV_INIT_DIST[0] <= (distance from adv2 to intersection) <= ADV_INIT_DIST[1]
+require adv3InitLane.road is egoManeuver.endLane.road
+terminate when (distance to egoSpawnPt) > TERM_DIST
+#or (distance from adv2 to adv2SpawnPt) > TERM_DIST + 40
+
+#################################
+# RECORDING #
+#################################
+
+record (ego in network.drivableRegion) as egoIsInDrivableRegion
+record (distance from ego to network.drivableRegion) as egoDistToDrivableRegion
+record (distance from ego to egoInitLane.group) as egoDistToEgoInitLane
+record (distance from ego to egoManeuver.endLane.group) as egoDistToEgoEndLane
+record (distance from ego to ego.lane.centerline) as egoDistToEgoLaneCenterline
+record (distance from ego to intersection) as egoDistToIntersection
+
+record (distance from ego to adv1) as egoDistToAdv1
+record (distance to egoSpawnPt) as egoDistToEgoSpawnPt
diff --git a/examples/dynamic_rulebook/multi_03/multi_03.sgraph b/examples/dynamic_rulebook/multi_03/multi_03.sgraph
new file mode 100644
index 0000000..f86898a
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_03/multi_03.sgraph
@@ -0,0 +1,26 @@
+# ID 0
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+2 on rule2 monitor
+3 on rule3 monitor
+4 on rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 on rule8 monitor
+9 on rule9 monitor
+10 on rule10 monitor
+# Edge list
+0 1
+0 2
+0 3
+0 4
+1 5
+2 5
+3 5
+4 5
+5 9
+5 10
+9 8
+10 8
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_03/multi_03_00.graph b/examples/dynamic_rulebook/multi_03/multi_03_00.graph
new file mode 100644
index 0000000..01bbba1
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_03/multi_03_00.graph
@@ -0,0 +1,17 @@
+# ID 0
+# Node list
+0 off rule0 monitor
+1 on rule1 monitor
+2 off rule2 monitor
+3 off rule3 monitor
+4 off rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 on rule8 monitor
+9 on rule9 monitor
+10 off rule10 monitor
+# Edge list
+1 5
+5 9
+9 8
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_03/multi_03_01.graph b/examples/dynamic_rulebook/multi_03/multi_03_01.graph
new file mode 100644
index 0000000..9d9091d
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_03/multi_03_01.graph
@@ -0,0 +1,23 @@
+# ID 1
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+2 on rule2 monitor
+3 on rule3 monitor
+4 on rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 off rule8 monitor
+9 off rule9 monitor
+10 on rule10 monitor
+# Edge list
+0 1
+0 2
+0 3
+0 4
+1 5
+2 5
+3 5
+4 5
+5 10
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_03/multi_03_02.graph b/examples/dynamic_rulebook/multi_03/multi_03_02.graph
new file mode 100644
index 0000000..3cfafcf
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_03/multi_03_02.graph
@@ -0,0 +1,18 @@
+# ID 2
+# Node list
+0 off rule0 monitor
+1 off rule1 monitor
+2 on rule2 monitor
+3 on rule3 monitor
+4 on rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 on rule8 monitor
+9 off rule9 monitor
+10 off rule10 monitor
+# Edge list
+2 5
+3 5
+4 5
+5 8
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_03/multi_03_rulebook.py b/examples/dynamic_rulebook/multi_03/multi_03_rulebook.py
new file mode 100644
index 0000000..a41cefa
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_03/multi_03_rulebook.py
@@ -0,0 +1,58 @@
+import numpy as np
+
+from verifai.rulebook import rulebook
+
+class rulebook_multi03(rulebook):
+ iteration = 0
+
+ def __init__(self, graph_path, rule_file, save_path=None, single_graph=False, using_sampler=-1, exploration_ratio=2.0):
+ rulebook.using_sampler = using_sampler
+ rulebook.exploration_ratio = exploration_ratio
+ super().__init__(graph_path, rule_file, single_graph=single_graph)
+ self.save_path = save_path
+
+ def evaluate(self, simulation):
+ # Extract trajectory information
+ positions = np.array(simulation.result.trajectory)
+ ego_dist_to_intersection = np.array(simulation.result.records["egoDistToIntersection"])
+
+ # Find switching points, i.e., ego has reached the intersection / ego has finished the right turn
+ switch_idx_1 = len(simulation.result.trajectory)
+ switch_idx_2 = len(simulation.result.trajectory)
+ for i in range(len(ego_dist_to_intersection)):
+ if ego_dist_to_intersection[i][1] == 0 and switch_idx_1 == len(simulation.result.trajectory):
+ switch_idx_1 = i
+ break
+ if switch_idx_1 < len(simulation.result.trajectory):
+ for i in reversed(range(switch_idx_1, len(ego_dist_to_intersection))):
+ if ego_dist_to_intersection[i][1] == 0:
+ switch_idx_2 = i + 1
+ break
+ assert switch_idx_1 <= switch_idx_2
+
+ # Evaluation
+ indices_0 = np.arange(0, switch_idx_1)
+ indices_1 = np.arange(switch_idx_1, switch_idx_2)
+ indices_2 = np.arange(switch_idx_2, len(simulation.result.trajectory))
+ #print('Indices:', indices_0, indices_1, indices_2)
+ if self.single_graph:
+ rho0 = self.evaluate_segment(simulation, 0, indices_0)
+ rho1 = self.evaluate_segment(simulation, 0, indices_1)
+ rho2 = self.evaluate_segment(simulation, 0, indices_2)
+ print('Actual rho:')
+ for r in rho0:
+ print(r, end=' ')
+ print()
+ for r in rho1:
+ print(r, end=' ')
+ print()
+ for r in rho2:
+ print(r, end=' ')
+ print()
+ rho = self.evaluate_segment(simulation, 0, np.arange(0, len(simulation.result.trajectory)))
+ return np.array([rho])
+ rho0 = self.evaluate_segment(simulation, 0, indices_0)
+ rho1 = self.evaluate_segment(simulation, 1, indices_1)
+ rho2 = self.evaluate_segment(simulation, 2, indices_2)
+ return np.array([rho0, rho1, rho2])
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_03/multi_03_spec.py b/examples/dynamic_rulebook/multi_03/multi_03_spec.py
new file mode 100644
index 0000000..123d6d5
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_03/multi_03_spec.py
@@ -0,0 +1,92 @@
+import numpy as np
+
+def rule0(simulation, indices): # A, 1: safe distance to ped
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_ped = positions[indices, [0], :] - positions[indices, [5], :]
+ distances_to_ped = np.linalg.norm(distances_to_ped, axis=1)
+ rho = np.min(distances_to_ped, axis=0) - 8
+ return rho
+
+def rule1(simulation, indices): # B, 1: safe distance to adv1
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [1], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule2(simulation, indices): # B, 2: safe distance to adv2
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [2], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule3(simulation, indices): # B, 3: safe distance to adv3
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [3], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule4(simulation, indices): # B, 4: safe distance to adv4
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [4], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule5(simulation, indices): # C: stay in drivable area
+ if indices.size == 0:
+ return 1
+ distance_to_drivable = np.array(simulation.result.records["egoDistToDrivableRegion"])
+ rho = -np.max(distance_to_drivable[indices], axis=0)[1]
+ return rho
+
+def rule6(simulation, indices): # D, 1: stay in the correct side of the road, before intersection
+ if indices.size == 0:
+ return 1
+ distance_to_lane_group = np.array(simulation.result.records["egoDistToEgoInitLane"])
+ rho = -np.max(distance_to_lane_group[indices], axis=0)[1]
+ return rho
+
+def rule7(simulation, indices): # D, 2: stay in the correct side of the road, after intersection
+ if indices.size == 0:
+ return 1
+ distance_to_lane_group = np.array(simulation.result.records["egoDistToEgoEndLane"])
+ rho = -np.max(distance_to_lane_group[indices], axis=0)[1]
+ return rho
+
+def rule8(simulation, indices): # F: lane keeping
+ if indices.size == 0:
+ return 1
+ distance_to_lane_center = np.array(simulation.result.records["egoDistToEgoLaneCenterline"])
+ rho = 0.4 - np.max(distance_to_lane_center[indices], axis=0)[1]
+ return rho
+
+def rule9(simulation, indices): # H, 1: reach intersection
+ if indices.size == 0:
+ return 1
+ if max(indices) < len(simulation.result.trajectory) - 1:
+ return 1
+ ego_dist_to_intersection = np.array(simulation.result.records["egoDistToIntersection"])
+ rho = -np.min(ego_dist_to_intersection[indices], axis=0)[1]
+ return rho
+
+def rule10(simulation, indices): # H, 2: finish right-turn
+ if indices.size == 0:
+ return 1
+ if max(indices) < len(simulation.result.trajectory) - 1:
+ return 1
+ ego_dist_to_end_lane = np.array(simulation.result.records["egoDistToEgoEndLane"])
+ rho = -np.min(ego_dist_to_end_lane[indices], axis=0)[1]
+ return rho
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_03/util/multi_03_analyze_diversity.py b/examples/dynamic_rulebook/multi_03/util/multi_03_analyze_diversity.py
new file mode 100644
index 0000000..ce7df45
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_03/util/multi_03_analyze_diversity.py
@@ -0,0 +1,48 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+directory = sys.argv[1]
+all_files = os.listdir(directory)
+all_files = [f for f in all_files if f.endswith('.csv') and f.startswith(sys.argv[2]+'.')]
+mode = sys.argv[3] # multi / single
+
+fig = plt.figure()
+ax = fig.add_subplot(projection='3d')
+count = 0
+adv1_speed = []
+adv2_speed = []
+adv_speed = []
+ego_speed = []
+for file in all_files:
+ infile = open(directory+'/'+file, 'r')
+ lines = infile.readlines()
+ if mode == 'single':
+ for i in range(1, len(lines)):
+ line = lines[i] #TODO: identify the counterexamples
+ ego_speed.append(float(line.split(',')[-13]))
+ adv_speed.append(float(line.split(',')[-14]))
+ adv2_speed.append(float(line.split(',')[-15]))
+ adv1_speed.append(float(line.split(',')[-16]))
+ else:
+ for i in range(1, len(lines), 3):
+ line1 = lines[i]
+ line2 = lines[i+1]
+ line3 = lines[i+2] #TODO: identify the counterexamples
+ ego_speed.append(float(line1.split(',')[-13]))
+ adv_speed.append(float(line1.split(',')[-14]))
+ adv2_speed.append(float(line1.split(',')[-15]))
+ adv1_speed.append(float(line1.split(',')[-16]))
+
+ax.scatter(ego_speed, adv_speed, adv2_speed)
+ax.set_xlabel('EGO_SPEED')
+ax.set_ylabel('ADV_SPEED')
+ax.set_zlabel('ADV2_SPEED')
+plt.savefig(directory+'/'+sys.argv[2]+'_scatter.png')
+
+print("Standard deviation of ego_speed:", np.std(ego_speed), len(ego_speed))
+print("Standard deviation of adv_speed:", np.std(adv_speed), len(adv_speed))
+print("Standard deviation of adv1_speed:", np.std(adv1_speed), len(adv1_speed))
+print("Standard deviation of adv2_speed:", np.std(adv2_speed), len(adv2_speed))
+print()
diff --git a/examples/dynamic_rulebook/multi_03/util/multi_03_collect_result.py b/examples/dynamic_rulebook/multi_03/util/multi_03_collect_result.py
new file mode 100644
index 0000000..2edeed4
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_03/util/multi_03_collect_result.py
@@ -0,0 +1,150 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import itertools
+
+infile = open(sys.argv[1], 'r') # *.txt
+mode = sys.argv[2] # multi / single
+order = sys.argv[3] # alternate / sequential
+
+# error weights
+result_count_0 = [[] for i in range(3)]
+result_count_1 = [[] for i in range(3)]
+result_count_2 = [[] for i in range(3)]
+# counterexample types
+counterexample_type_0 = [{} for i in range(3)]
+counterexample_type_1 = [{} for i in range(3)]
+counterexample_type_2 = [{} for i in range(3)]
+curr_source = 0
+lines = infile.readlines()
+infile.close()
+
+for i in range(len(lines)):
+ if mode == 'multi':
+ if 'RHO' in lines[i]:
+ line = lines[i+1].strip().split(' ')
+ val1 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val1.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val1) == 11, 'Invalid length of rho'
+ #print('Rho 0:', val_print[1], val_print[5], val_print[9], val_print[8])
+ result_count_0[curr_source].append(val1[1]*8 + val1[5]*4 + val1[9]*2 + val1[8]*1)
+ if tuple(1*np.array([val1[1], val1[5], val1[9], val1[8]])) in counterexample_type_0[curr_source]:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[1], val1[5], val1[9], val1[8]]))] += 1
+ else:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[1], val1[5], val1[9], val1[8]]))] = 1
+
+ line = lines[i+2].strip().split(' ')
+ val2 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val2.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val2) == 11, 'Invalid length of rho'
+ #print('Rho 1:', val_print[0], val_print[1], val_print[2], val_print[3], val_print[4], val_print[5], val_print[10])
+ result_count_1[curr_source].append(val2[0]*64 + val2[1]*4 + val2[2]*4 + val2[3]*4 + val2[4]*4 + val2[5]*2 + val2[10]*1)
+ if tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[4], val2[5], val2[10]])) in counterexample_type_1[curr_source]:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[4], val2[5], val2[10]]))] += 1
+ else:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[4], val2[5], val2[10]]))] = 1
+
+ line = lines[i+3].strip().split(' ')
+ val3 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val3.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val3) == 11, 'Invalid length of rho'
+ #print('Rho 2:', val_print[2], val_print[3], val_print[4], val_print[5], val_print[8], '\n')
+ result_count_2[curr_source].append(val3[2]*4 + val3[3]*4 + val3[4]*4 + val3[5]*2 + val3[8]*1)
+ if tuple(1*np.array([val3[2], val3[3], val3[4], val3[5], val3[8]])) in counterexample_type_2[curr_source]:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[2], val3[3], val3[4], val3[5], val3[8]]))] += 1
+ else:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[2], val3[3], val3[4], val3[5], val3[8]]))] = 1
+
+ if order == '-1':
+ curr_source = curr_source + 1 if curr_source < 2 else 0
+ else:
+ if 'Actual rho' in lines[i]:
+ line = lines[i+1].strip().split(' ')
+ val1 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val1.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val1) == 11, 'Invalid length of rho'
+ #print('Rho 0:', val_print[1], val_print[5], val_print[9], val_print[8])
+ result_count_0[curr_source].append(val1[1]*8 + val1[5]*4 + val1[9]*2 + val1[8]*1)
+ if tuple(1*np.array([val1[1], val1[5], val1[9], val1[8]])) in counterexample_type_0[curr_source]:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[1], val1[5], val1[9], val1[8]]))] += 1
+ else:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[1], val1[5], val1[9], val1[8]]))] = 1
+
+ line = lines[i+2].strip().split(' ')
+ val2 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val2.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val2) == 11, 'Invalid length of rho'
+ #print('Rho 1:', val_print[0], val_print[1], val_print[2], val_print[3], val_print[4], val_print[5], val_print[10])
+ result_count_1[curr_source].append(val2[0]*64 + val2[1]*4 + val2[2]*4 + val2[3]*4 + val2[4]*4 + val2[5]*2 + val2[10]*1)
+ if tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[4], val2[5], val2[10]])) in counterexample_type_1[curr_source]:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[4], val2[5], val2[10]]))] += 1
+ else:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[4], val2[5], val2[10]]))] = 1
+
+ line = lines[i+3].strip().split(' ')
+ val3 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val3.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val3) == 11, 'Invalid length of rho'
+ #print('Rho 2:', val_print[2], val_print[3], val_print[4], val_print[5], val_print[8], '\n')
+ result_count_2[curr_source].append(val3[2]*4 + val3[3]*4 + val3[4]*4 + val3[5]*2 + val3[8]*1)
+ if tuple(1*np.array([val3[2], val3[3], val3[4], val3[5], val3[8]])) in counterexample_type_2[curr_source]:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[2], val3[3], val3[4], val3[5], val3[8]]))] += 1
+ else:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[2], val3[3], val3[4], val3[5], val3[8]]))] = 1
+
+ if order == '-1':
+ curr_source = curr_source + 1 if curr_source < 2 else 0
+
+print('Error weights')
+print('segment 0:')
+for i in range(1):
+ print('average:', np.mean(result_count_0[i]), 'max:', np.max(result_count_0[i]), 'percentage:', float(np.count_nonzero(result_count_0[i])/len(result_count_0[i])), result_count_0[i])
+print('segment 1:')
+for i in range(1):
+ print('average:', np.mean(result_count_1[i]), 'max:', np.max(result_count_1[i]), 'percentage:', float(np.count_nonzero(result_count_1[i])/len(result_count_1[i])), result_count_1[i])
+print('segment 2:')
+for i in range(1):
+ print('average:', np.mean(result_count_2[i]), 'max:', np.max(result_count_2[i]), 'percentage:', float(np.count_nonzero(result_count_2[i])/len(result_count_2[i])), result_count_2[i])
+
+print('\nCounterexample types')
+print('segment 0:')
+for i in range(1):
+ print('Types:', len(counterexample_type_0[i]))
+ for key, value in reversed(sorted(counterexample_type_0[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print('segment 1:')
+for i in range(1):
+ print('Types:', len(counterexample_type_1[i]))
+ for key, value in reversed(sorted(counterexample_type_1[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print('segment 2:')
+for i in range(1):
+ print('Types:', len(counterexample_type_2[i]))
+ for key, value in reversed(sorted(counterexample_type_2[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print()
diff --git a/examples/dynamic_rulebook/multi_04/multi_04.py b/examples/dynamic_rulebook/multi_04/multi_04.py
new file mode 100644
index 0000000..64076e5
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_04/multi_04.py
@@ -0,0 +1,49 @@
+import sys
+import os
+sys.path.append(os.path.abspath("."))
+import random
+import numpy as np
+
+from multi import *
+from multi_04_rulebook import rulebook_multi04
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--scenic-path', '-sp', type=str, default='uberCrashNewton.scenic',
+ help='Path to Scenic script')
+ parser.add_argument('--graph-path', '-gp', type=str, default=None,
+ help='Path to graph file')
+ parser.add_argument('--rule-path', '-rp', type=str, default=None,
+ help='Path to rule file')
+ parser.add_argument('--output-dir', '-o', type=str, default=None,
+ help='Directory to save output trajectories')
+ parser.add_argument('--output-csv-dir', '-co', type=str, default=None,
+ help='Directory to save output error tables (csv files)')
+ parser.add_argument('--parallel', action='store_true')
+ parser.add_argument('--num-workers', type=int, default=5, help='Number of parallel workers')
+ parser.add_argument('--sampler-type', '-s', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--experiment-name', '-e', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--model', '-m', type=str, default='scenic.simulators.newtonian.driving_model')
+ parser.add_argument('--headless', action='store_true')
+ parser.add_argument('--n-iters', '-n', type=int, default=None, help='Number of simulations to run')
+ parser.add_argument('--max-time', type=int, default=None, help='Maximum amount of time to run simulations')
+ parser.add_argument('--single-graph', action='store_true', help='Only a unified priority graph')
+ parser.add_argument('--seed', type=int, default=0, help='Random seed')
+ parser.add_argument('--using-sampler', type=int, default=-1, help='Assigning sampler to use')
+ parser.add_argument('--max-simulation-steps', type=int, default=300, help='Maximum number of simulation steps')
+ args = parser.parse_args()
+ if args.n_iters is None and args.max_time is None:
+ raise ValueError('At least one of --n-iters or --max-time must be set')
+
+ random.seed(args.seed)
+ np.random.seed(args.seed)
+
+ rb = rulebook_multi04(args.graph_path, args.rule_path, save_path=args.output_dir, single_graph=args.single_graph, using_sampler=args.using_sampler)
+ run_experiments(args.scenic_path, rulebook=rb,
+ parallel=args.parallel, model=args.model,
+ sampler_type=args.sampler_type, headless=args.headless,
+ num_workers=args.num_workers, output_dir=args.output_csv_dir, experiment_name=args.experiment_name,
+ max_time=args.max_time, n_iters=args.n_iters, max_steps=args.max_simulation_steps)
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_04/multi_04.scenic b/examples/dynamic_rulebook/multi_04/multi_04.scenic
new file mode 100644
index 0000000..2cce216
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_04/multi_04.scenic
@@ -0,0 +1,165 @@
+"""
+TITLE: Multi 04
+AUTHOR: Kai-Chun Chang, kaichunchang@berkeley.edu"""
+
+#################################
+# MAP AND MODEL #
+#################################
+
+param map = localPath('../maps/Town05.xodr')
+param carla_map = 'Town05'
+param N = 13
+model scenic.domains.driving.model
+
+#################################
+# CONSTANTS #
+#################################
+
+MODEL = 'vehicle.lincoln.mkz_2017'
+
+INIT_DIST = [15, 20]
+v3_DIST = -10
+param VEHICLE_SPEED = VerifaiRange(8, 10)
+param VEHICLE_BRAKE = VerifaiRange(0.8, 1.0)
+
+SAFETY_DIST = 8
+param ARRIVE_INTERSECTION_DIST = VerifaiRange(2, 5)
+TERM_DIST = 50
+ARRIVING_ORDER = []
+HAS_PASSED = [False, False, False, False]
+PASSING_ORDER = []
+
+#################################
+# AGENT BEHAVIORS #
+#################################
+
+def CanEnter(id):
+ for i in range(len(ARRIVING_ORDER)):
+ if ARRIVING_ORDER[i] == id:
+ return True
+ if HAS_PASSED[ARRIVING_ORDER[i]] == False:
+ return False
+
+behavior VehicleBehavior(trajectory, id):
+ wait_flag = False # if the vehicle has joined the waiting list
+ enter_flag = False # if the vehicle has entered the intersection
+ leave_flag = False # if the vehicle has passed the intersection
+ if id == 0:
+ ARRIVING_ORDER.clear()
+ PASSING_ORDER.clear()
+ HAS_PASSED[id] = False
+ try:
+ do FollowTrajectoryBehavior(target_speed=globalParameters.VEHICLE_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.VEHICLE_SPEED)
+ #interrupt when (distance from self to intersection) < globalParameters.ARRIVE_INTERSECTION_DIST and not CanEnter(id):
+ # take SetBrakeAction(globalParameters.VEHICLE_BRAKE)
+ interrupt when (distance from self to intersection) < globalParameters.ARRIVE_INTERSECTION_DIST and not wait_flag:
+ ARRIVING_ORDER.append(id)
+ #print("Vehicle", id, "is waiting", ARRIVING_ORDER)
+ wait_flag = True
+ interrupt when (distance from self to intersection) == 0 and wait_flag and not enter_flag:
+ #print("Vehicle", id, "is entering")
+ enter_flag = True
+ interrupt when (distance from self to intersection) > 0 and enter_flag and not leave_flag:
+ #print("Vehicle", id, "has passed")
+ leave_flag = True
+ HAS_PASSED[id] = True
+ PASSING_ORDER.append(id)
+ interrupt when withinDistanceToAnyObjs(self, SAFETY_DIST):
+ take SetBrakeAction(globalParameters.VEHICLE_BRAKE)
+
+behavior FollowBehavior(trajectory, id, front):
+ wait_flag = False # if the vehicle has joined the waiting list
+ enter_flag = False # if the vehicle has entered the intersection
+ leave_flag = False # if the vehicle has passed the intersection
+ if id == 0:
+ ARRIVING_ORDER.clear()
+ HAS_PASSED[id] = False
+ try:
+ do FollowTrajectoryBehavior(target_speed=globalParameters.VEHICLE_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.VEHICLE_SPEED)
+ #interrupt when (distance from self to intersection) < globalParameters.ARRIVE_INTERSECTION_DIST and not CanEnter(id):
+ # take SetBrakeAction(globalParameters.VEHICLE_BRAKE)
+ interrupt when (distance from self to intersection) < globalParameters.ARRIVE_INTERSECTION_DIST and not wait_flag:
+ ARRIVING_ORDER.append(id)
+ #print("Vehicle", id, "is waiting", ARRIVING_ORDER)
+ wait_flag = True
+ interrupt when (distance from self to intersection) == 0 and wait_flag and not enter_flag:
+ #print("Vehicle", id, "is entering")
+ enter_flag = True
+ interrupt when (distance from self to intersection) > 0 and enter_flag and not leave_flag:
+ #print("Vehicle", id, "has passed")
+ leave_flag = True
+ HAS_PASSED[id] = True
+ PASSING_ORDER.append(id)
+ interrupt when (distance from self to front) < SAFETY_DIST:
+ take SetBrakeAction(globalParameters.VEHICLE_BRAKE)
+ interrupt when withinDistanceToAnyObjs(self, SAFETY_DIST):
+ take SetBrakeAction(globalParameters.VEHICLE_BRAKE)
+
+#################################
+# SPATIAL RELATIONS #
+#################################
+
+intersection = Uniform(*filter(lambda i: i.is4Way, network.intersections))
+
+# v0: straight from S to N
+v0Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, intersection.maneuvers))
+v0InitLane = v0Maneuver.startLane
+v0Trajectory = [v0InitLane, v0Maneuver.connectingLane, v0Maneuver.endLane]
+v0SpawnPt = new OrientedPoint in v0InitLane.centerline
+
+# v1: straight from W to E or E to W
+v1InitLane = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT,
+ Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, v0InitLane.maneuvers)).conflictingManeuvers)).startLane
+v1Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, v1InitLane.maneuvers))
+v1Trajectory = [v1InitLane, v1Maneuver.connectingLane, v1Maneuver.endLane]
+v1SpawnPt = new OrientedPoint in v1InitLane.centerline
+
+# v2: straight from E to W or W to E (reverse to v1)
+v2InitLane = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, v1Maneuver.reverseManeuvers)).startLane
+v2Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, v2InitLane.maneuvers))
+v2Trajectory = [v2InitLane, v2Maneuver.connectingLane, v2Maneuver.endLane]
+v2SpawnPt = new OrientedPoint in v2InitLane.centerline
+
+# v3: behind v0
+v3InitLane = v0InitLane
+v3Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, v3InitLane.maneuvers))
+v3Trajectory = [v3InitLane, v3Maneuver.connectingLane, v3Maneuver.endLane]
+
+#################################
+# SCENARIO SPECIFICATION #
+#################################
+
+ego = new Car at v0SpawnPt,
+ with blueprint MODEL,
+ with behavior VehicleBehavior(v0Trajectory, 0)
+
+v1 = new Car at v1SpawnPt,
+ with blueprint MODEL,
+ with behavior VehicleBehavior(v1Trajectory, 1)
+
+v2 = new Car at v2SpawnPt,
+ with blueprint MODEL,
+ with behavior VehicleBehavior(v2Trajectory, 2)
+
+v3 = new Car following roadDirection for v3_DIST,
+ with blueprint MODEL,
+ with behavior FollowBehavior(v3Trajectory, 3, ego)
+
+require INIT_DIST[0] <= (distance from ego to intersection) <= INIT_DIST[1]
+require INIT_DIST[0] <= (distance from v1 to intersection) <= INIT_DIST[1]
+require INIT_DIST[0] <= (distance from v2 to intersection) <= INIT_DIST[1]
+terminate when (distance to v0SpawnPt) > TERM_DIST and HAS_PASSED[0] and HAS_PASSED[1] and HAS_PASSED[2] and HAS_PASSED[3]
+
+#################################
+# RECORDING #
+#################################
+
+record final ARRIVING_ORDER as arrivingOrder
+record final PASSING_ORDER as passingOrder
+record final HAS_PASSED as hasPassed
+record ((distance from ego to intersection) == 0) as v0IsInIntersection
+record ((distance from v1 to intersection) == 0) as v1IsInIntersection
+record ((distance from v2 to intersection) == 0) as v2IsInIntersection
+record ((distance from v3 to intersection) == 0) as v3IsInIntersection
diff --git a/examples/dynamic_rulebook/multi_04/multi_04_00.graph b/examples/dynamic_rulebook/multi_04/multi_04_00.graph
new file mode 100644
index 0000000..ed6cc38
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_04/multi_04_00.graph
@@ -0,0 +1,52 @@
+# ID 0
+# Node list
+0 on ruleA01 monitor
+1 on ruleA02 monitor
+2 on ruleA03 monitor
+3 on ruleA12 monitor
+4 on ruleA13 monitor
+5 on ruleA23 monitor
+6 on ruleB0 monitor
+7 on ruleB1 monitor
+8 on ruleB2 monitor
+9 on ruleB3 monitor
+10 on ruleC0 monitor
+11 on ruleC1 monitor
+12 on ruleC2 monitor
+# Edge list
+0 6
+1 6
+2 6
+3 6
+4 6
+5 6
+0 7
+1 7
+2 7
+3 7
+4 7
+5 7
+0 8
+1 8
+2 8
+3 8
+4 8
+5 8
+0 9
+1 9
+2 9
+3 9
+4 9
+5 9
+6 10
+6 11
+6 12
+7 10
+7 11
+7 12
+8 10
+8 11
+8 12
+9 10
+9 11
+9 12
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_04/multi_04_rulebook.py b/examples/dynamic_rulebook/multi_04/multi_04_rulebook.py
new file mode 100644
index 0000000..7a20853
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_04/multi_04_rulebook.py
@@ -0,0 +1,48 @@
+import numpy as np
+
+from verifai.rulebook import rulebook
+
+class rulebook_multi04(rulebook):
+ iteration = 0
+
+ def __init__(self, graph_path, rule_file, save_path=None, single_graph=False, using_sampler=-1):
+ rulebook.using_sampler = using_sampler
+ super().__init__(graph_path, rule_file, single_graph=single_graph)
+ self.save_path = save_path
+
+ def evaluate(self, simulation):
+ # Extract trajectory information
+ v0_is_in_intersection = np.array(simulation.result.records["v0IsInIntersection"])
+ v0_is_in_intersection = v0_is_in_intersection[:, 1]
+ v1_is_in_intersection = np.array(simulation.result.records["v1IsInIntersection"])
+ v1_is_in_intersection = v1_is_in_intersection[:, 1]
+ v2_is_in_intersection = np.array(simulation.result.records["v2IsInIntersection"])
+ v2_is_in_intersection = v2_is_in_intersection[:, 1]
+ v3_is_in_intersection = np.array(simulation.result.records["v3IsInIntersection"])
+ v3_is_in_intersection = v3_is_in_intersection[:, 1]
+
+ # Find indices for each rule
+ indices_A01 = np.where(v0_is_in_intersection & v1_is_in_intersection)[0]
+ indices_A02 = np.where(v0_is_in_intersection & v2_is_in_intersection)[0]
+ indices_A03 = np.where(v0_is_in_intersection & v3_is_in_intersection)[0]
+ indices_A12 = np.where(v1_is_in_intersection & v2_is_in_intersection)[0]
+ indices_A13 = np.where(v1_is_in_intersection & v3_is_in_intersection)[0]
+ indices_A23 = np.where(v2_is_in_intersection & v3_is_in_intersection)[0]
+
+ # Evaluation
+ rho_A01 = self.evaluate_rule(simulation, rule_id=0, graph_idx=0, indices=indices_A01)
+ rho_A02 = self.evaluate_rule(simulation, rule_id=1, graph_idx=0, indices=indices_A02)
+ rho_A03 = self.evaluate_rule(simulation, rule_id=2, graph_idx=0, indices=indices_A03)
+ rho_A12 = self.evaluate_rule(simulation, rule_id=3, graph_idx=0, indices=indices_A12)
+ rho_A13 = self.evaluate_rule(simulation, rule_id=4, graph_idx=0, indices=indices_A13)
+ rho_A23 = self.evaluate_rule(simulation, rule_id=5, graph_idx=0, indices=indices_A23)
+ rho_B0 = self.evaluate_rule(simulation, rule_id=6, graph_idx=0)
+ rho_B1 = self.evaluate_rule(simulation, rule_id=7, graph_idx=0)
+ rho_B2 = self.evaluate_rule(simulation, rule_id=8, graph_idx=0)
+ rho_B3 = self.evaluate_rule(simulation, rule_id=9, graph_idx=0)
+ rho_C0 = self.evaluate_rule(simulation, rule_id=10, graph_idx=0)
+ rho_C1 = self.evaluate_rule(simulation, rule_id=11, graph_idx=0)
+ rho_C2 = self.evaluate_rule(simulation, rule_id=12, graph_idx=0)
+ rho = np.array([rho_A01, rho_A02, rho_A03, rho_A12, rho_A13, rho_A23, rho_B0, rho_B1, rho_B2, rho_B3, rho_C0, rho_C1, rho_C2])
+ return np.array([rho])
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_04/multi_04_spec.py b/examples/dynamic_rulebook/multi_04/multi_04_spec.py
new file mode 100644
index 0000000..2f23e36
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_04/multi_04_spec.py
@@ -0,0 +1,121 @@
+import numpy as np
+
+def ruleA01(simulation, indices): # A, 0, 1: safe distance from v0 to v1
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances = positions[indices, [0], :] - positions[indices, [1], :]
+ distances = np.linalg.norm(distances, axis=1)
+ rho = np.min(distances, axis=0) - 8
+ return rho
+
+def ruleA02(simulation, indices): # A, 0, 2: safe distance from v0 to v2
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances = positions[indices, [0], :] - positions[indices, [2], :]
+ distances = np.linalg.norm(distances, axis=1)
+ rho = np.min(distances, axis=0) - 8
+ return rho
+
+def ruleA03(simulation, indices): # A, 0, 3: safe distance from v0 to v3
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances = positions[indices, [0], :] - positions[indices, [3], :]
+ distances = np.linalg.norm(distances, axis=1)
+ rho = np.min(distances, axis=0) - 8
+ return rho
+
+def ruleA12(simulation, indices): # A, 1, 2: safe distance from v1 to v2
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances = positions[indices, [1], :] - positions[indices, [2], :]
+ distances = np.linalg.norm(distances, axis=1)
+ rho = np.min(distances, axis=0) - 8
+ return rho
+
+def ruleA13(simulation, indices): # A, 1, 3: safe distance from v1 to v3
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances = positions[indices, [1], :] - positions[indices, [3], :]
+ distances = np.linalg.norm(distances, axis=1)
+ rho = np.min(distances, axis=0) - 8
+ return rho
+
+def ruleA23(simulation, indices): # A, 2, 3: safe distance from v2 to v3
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances = positions[indices, [2], :] - positions[indices, [3], :]
+ distances = np.linalg.norm(distances, axis=1)
+ rho = np.min(distances, axis=0) - 8
+ return rho
+
+def ruleB0(simulation, indices): # B, 0: v0 successfully passes the intersection
+ has_passed = simulation.result.records["hasPassed"]
+ if has_passed[0]:
+ return 1
+ return -1 #TODO
+
+def ruleB1(simulation, indices): # B, 1: v1 successfully passes the intersection
+ has_passed = simulation.result.records["hasPassed"]
+ if has_passed[1]:
+ return 1
+ return -1 #TODO
+
+def ruleB2(simulation, indices): # B, 2: v2 successfully passes the intersection
+ has_passed = simulation.result.records["hasPassed"]
+ if has_passed[2]:
+ return 1
+ return -1 #TODO
+
+def ruleB3(simulation, indices): # B, 3: v3 successfully passes the intersection
+ has_passed = simulation.result.records["hasPassed"]
+ if has_passed[3]:
+ return 1
+ return -1 #TODO
+
+def ruleC0(simulation, indices): # C, 0: the first pair of ordering
+ arriving_order = simulation.result.records["arrivingOrder"]
+ passing_order = simulation.result.records["passingOrder"]
+ idx_0 = 10
+ idx_1 = 10
+ for i in range(len(passing_order)):
+ if passing_order[i] == arriving_order[0]:
+ idx_0 = i
+ elif passing_order[i] == arriving_order[1]:
+ idx_1 = i
+ if idx_0 < idx_1:
+ return 1
+ return -1
+
+def ruleC1(simulation, indices): # C, 1: the second pair of ordering
+ arriving_order = simulation.result.records["arrivingOrder"]
+ passing_order = simulation.result.records["passingOrder"]
+ idx_1 = 10
+ idx_2 = 10
+ for i in range(len(passing_order)):
+ if passing_order[i] == arriving_order[1]:
+ idx_1 = i
+ elif passing_order[i] == arriving_order[2]:
+ idx_2 = i
+ if idx_1 < idx_2:
+ return 1
+ return -1
+
+def ruleC2(simulation, indices): # C, 2: the third pair of ordering
+ arriving_order = simulation.result.records["arrivingOrder"]
+ passing_order = simulation.result.records["passingOrder"]
+ idx_2 = 10
+ idx_3 = 10
+ for i in range(len(passing_order)):
+ if passing_order[i] == arriving_order[2]:
+ idx_2 = i
+ elif passing_order[i] == arriving_order[3]:
+ idx_3 = i
+ if idx_2 < idx_3:
+ return 1
+ return -1
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_04/util/multi_04_analyze_diversity.py b/examples/dynamic_rulebook/multi_04/util/multi_04_analyze_diversity.py
new file mode 100644
index 0000000..3bba728
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_04/util/multi_04_analyze_diversity.py
@@ -0,0 +1,38 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+directory = sys.argv[1]
+all_files = os.listdir(directory)
+all_files = [f for f in all_files if f.endswith('.csv') and f.startswith(sys.argv[2]+'.')]
+mode = sys.argv[3] # multi / single
+
+fig = plt.figure()
+ax = fig.add_subplot(projection='3d')
+count = 0
+speed = []
+brake = []
+arrving_dist = []
+
+for file in all_files:
+ infile = open(directory+'/'+file, 'r')
+ lines = infile.readlines()
+ for i in range(1, len(lines)):
+ line = lines[i]
+ rhos = np.array(line.split(',')[-13:-1]).astype(float)
+ if np.any(rhos < 0):
+ speed.append(float(line.split(',')[-14]))
+ brake.append(float(line.split(',')[-15]))
+ arrving_dist.append(float(line.split(',')[-16]))
+
+ax.scatter(speed, brake, arrving_dist)
+ax.set_xlabel('SPEED')
+ax.set_ylabel('BRAKE')
+ax.set_zlabel('ARRIVING DISTANCE')
+plt.savefig(directory+'/'+sys.argv[2]+'_scatter.png')
+
+print("Standard deviation of speed:", np.std(speed))
+print("Standard deviation of brake:", np.std(brake))
+print("Standard deviation of arrving_dist:", np.std(arrving_dist))
+print()
diff --git a/examples/dynamic_rulebook/multi_04/util/multi_04_collect_result.py b/examples/dynamic_rulebook/multi_04/util/multi_04_collect_result.py
new file mode 100644
index 0000000..65616d1
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_04/util/multi_04_collect_result.py
@@ -0,0 +1,40 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import itertools
+
+infile = open(sys.argv[1], 'r') # *.txt
+mode = sys.argv[2] # multi / single
+order = sys.argv[3] # alternate / sequential
+
+# error weights
+result_count = []
+# counterexample types
+counterexample_type = {}
+lines = infile.readlines()
+infile.close()
+
+for i in range(len(lines)):
+ if mode == 'multi':
+ if 'RHO' in lines[i]:
+ line = lines[i+1].strip().split(' ')
+ val = []
+ for s in line:
+ if s != '':
+ val.append(float(s) < 0)
+ assert len(val) == 13, 'Invalid length of rho'
+ result_count.append((val[0] + val[1] + val[2] + val[3] + val[4] + val[5])*128 + (val[6] + val[7] + val[8] + val[9])*8 + val[10] + val[11] + val[12])
+ if tuple(1*np.array([val[0], val[1], val[2], val[3], val[4], val[5], val[6], val[7], val[8], val[9], val[10], val[11], val[12]])) in counterexample_type:
+ counterexample_type[tuple(1*np.array([val[0], val[1], val[2], val[3], val[4], val[5], val[6], val[7], val[8], val[9], val[10], val[11], val[12]]))] += 1
+ else:
+ counterexample_type[tuple(1*np.array([val[0], val[1], val[2], val[3], val[4], val[5], val[6], val[7], val[8], val[9], val[10], val[11], val[12]]))] = 1
+
+print('Error weights:')
+print('average:', float(sum(result_count)/len(result_count)), 'max:', np.max(result_count), 'percentage:', float(np.count_nonzero(result_count)/len(result_count)), result_count)
+
+print('\nCounterexample types')
+print('Types:', len(counterexample_type))
+for key, value in reversed(sorted(counterexample_type.items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print()
diff --git a/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left.py b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left.py
new file mode 100644
index 0000000..a2fcade
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left.py
@@ -0,0 +1,51 @@
+import sys
+import os
+sys.path.append(os.path.abspath("."))
+import random
+import numpy as np
+
+from multi import *
+from multi_verifai2left_rulebook import rulebook_multileft
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--scenic-path', '-sp', type=str, default='uberCrashNewton.scenic',
+ help='Path to Scenic script')
+ parser.add_argument('--graph-path', '-gp', type=str, default=None,
+ help='Path to graph file')
+ parser.add_argument('--rule-path', '-rp', type=str, default=None,
+ help='Path to rule file')
+ parser.add_argument('--output-dir', '-o', type=str, default=None,
+ help='Directory to save output trajectories')
+ parser.add_argument('--output-csv-dir', '-co', type=str, default=None,
+ help='Directory to save output error tables (csv files)')
+ parser.add_argument('--parallel', action='store_true')
+ parser.add_argument('--num-workers', type=int, default=5, help='Number of parallel workers')
+ parser.add_argument('--sampler-type', '-s', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--experiment-name', '-e', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--model', '-m', type=str, default='scenic.simulators.newtonian.driving_model')
+ parser.add_argument('--headless', action='store_true')
+ parser.add_argument('--n-iters', '-n', type=int, default=None, help='Number of simulations to run')
+ parser.add_argument('--max-time', type=int, default=None, help='Maximum amount of time to run simulations')
+ parser.add_argument('--single-graph', action='store_true', help='Only a unified priority graph')
+ parser.add_argument('--seed', type=int, default=0, help='Random seed')
+ parser.add_argument('--using-sampler', type=int, default=-1, help='Assigning sampler to use')
+ parser.add_argument('--max-simulation-steps', type=int, default=300, help='Maximum number of simulation steps')
+ parser.add_argument('--exploration-ratio', type=float, default=2.0, help='Exploration ratio')
+ args = parser.parse_args()
+ if args.n_iters is None and args.max_time is None:
+ raise ValueError('At least one of --n-iters or --max-time must be set')
+
+ random.seed(args.seed)
+ np.random.seed(args.seed)
+
+ rb = rulebook_multileft(args.graph_path, args.rule_path, save_path=args.output_dir, single_graph=args.single_graph,
+ using_sampler=args.using_sampler, exploration_ratio=args.exploration_ratio)
+ run_experiments(args.scenic_path, rulebook=rb,
+ parallel=args.parallel, model=args.model,
+ sampler_type=args.sampler_type, headless=args.headless,
+ num_workers=args.num_workers, output_dir=args.output_csv_dir, experiment_name=args.experiment_name,
+ max_time=args.max_time, n_iters=args.n_iters, max_steps=args.max_simulation_steps)
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left.scenic b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left.scenic
new file mode 100644
index 0000000..7d7833d
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left.scenic
@@ -0,0 +1,137 @@
+"""
+TITLE: Verifai 2.0 Left Turn
+AUTHOR: Kai-Chun Chang, kaichunchang@berkeley.edu
+"""
+
+#################################
+# MAP AND MODEL #
+#################################
+
+param map = localPath('../maps/Town05.xodr')
+param carla_map = 'Town05'
+model scenic.domains.driving.model
+
+#################################
+# CONSTANTS #
+#################################
+
+MODEL = 'vehicle.lincoln.mkz_2017' #'vehicle.toyota.prius'
+MODEL_ADV = 'vehicle.lincoln.mkz_2017'
+
+EGO_INIT_DIST = [30, 40]
+param EGO_SPEED = VerifaiRange(7, 10)
+param EGO_BRAKE = VerifaiRange(0.8, 1.0)
+
+param ADV1_DIST = VerifaiRange(6, 10)
+ADV_INIT_DIST = [15, 25]
+param ADV_SPEED = VerifaiRange(5, 8)
+
+PED_MIN_SPEED = 1.0
+PED_THRESHOLD = 20
+PED_FINAL_SPEED = 1.0
+
+SAFETY_DIST = 8
+CRASH_DIST = 5
+TERM_DIST = 80
+
+#################################
+# AGENT BEHAVIORS #
+#################################
+
+behavior EgoBehavior(trajectory):
+ try:
+ do FollowTrajectoryBehavior(target_speed=globalParameters.EGO_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.EGO_SPEED)
+ interrupt when withinDistanceToAnyObjs(self, SAFETY_DIST):
+ take SetBrakeAction(globalParameters.EGO_BRAKE)
+
+behavior Adv1Behavior(trajectory):
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+
+behavior Adv2Behavior(trajectory):
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+
+behavior Adv3Behavior(trajectory):
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+
+#################################
+# SPATIAL RELATIONS #
+#################################
+
+intersection = Uniform(*filter(lambda i: i.is4Way, network.intersections))
+
+# ego: left turn from S to W
+egoManeuver = Uniform(*filter(lambda m: m.type is ManeuverType.LEFT_TURN, intersection.maneuvers))
+egoInitLane = egoManeuver.startLane
+egoTrajectory = [egoInitLane, egoManeuver.connectingLane, egoManeuver.endLane]
+egoSpawnPt = new OrientedPoint in egoInitLane.centerline
+
+# adv1: straight from S to N
+adv1InitLane = egoInitLane
+adv1Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv1InitLane.maneuvers))
+adv1Trajectory = [adv1InitLane, adv1Maneuver.connectingLane, adv1Maneuver.endLane]
+
+# adv2: straight from W to E
+adv2InitLane = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT,
+ Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, egoInitLane.maneuvers)).conflictingManeuvers)).startLane
+adv2Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv2InitLane.maneuvers))
+adv2Trajectory = [adv2InitLane, adv2Maneuver.connectingLane, adv2Maneuver.endLane]
+adv2SpawnPt = new OrientedPoint in adv2InitLane.centerline
+
+# adv3: straight from E to W
+adv3InitLane = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv2Maneuver.reverseManeuvers)).startLane
+adv3Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv3InitLane.maneuvers))
+adv3Trajectory = [adv3InitLane, adv3Maneuver.connectingLane, adv3Maneuver.endLane]
+adv3SpawnPt = new OrientedPoint in adv3InitLane.centerline
+
+#################################
+# SCENARIO SPECIFICATION #
+#################################
+
+ego = new Car at egoSpawnPt,
+ with blueprint MODEL,
+ with behavior EgoBehavior(egoTrajectory)
+
+adv1 = new Car following roadDirection for globalParameters.ADV1_DIST,
+ with blueprint MODEL_ADV,
+ with behavior Adv1Behavior(adv1Trajectory)
+
+adv2 = new Car at adv2SpawnPt,
+ with blueprint MODEL_ADV,
+ with behavior Adv2Behavior(adv2Trajectory)
+
+adv3 = new Car at adv3SpawnPt,
+ with blueprint MODEL_ADV,
+ with behavior Adv3Behavior(adv3Trajectory)
+
+require EGO_INIT_DIST[0] <= (distance to intersection) <= EGO_INIT_DIST[1]
+require ADV_INIT_DIST[0] <= (distance from adv2 to intersection) <= ADV_INIT_DIST[1]
+require ADV_INIT_DIST[0] <= (distance from adv3 to intersection) <= ADV_INIT_DIST[1]
+require adv2InitLane.road is egoManeuver.endLane.road
+terminate when (distance to egoSpawnPt) > TERM_DIST
+
+#################################
+# RECORDING #
+#################################
+
+record (ego in network.drivableRegion) as egoIsInDrivableRegion
+record (distance from ego to network.drivableRegion) as egoDistToDrivableRegion
+record (distance from ego to egoInitLane.group) as egoDistToEgoInitLane
+record (distance from ego to egoManeuver.endLane.group) as egoDistToEgoEndLane
+record (distance from ego to ego.lane.centerline) as egoDistToEgoLaneCenterline
+record (distance from ego to intersection) as egoDistToIntersection
+
+record (distance from ego to adv1) as egoDistToAdv1
+record (distance to egoSpawnPt) as egoDistToEgoSpawnPt
+
+record ego._boundingPolygon as egoPoly
+record adv1._boundingPolygon as adv1Poly
+record adv2._boundingPolygon as adv2Poly
+record adv3._boundingPolygon as adv3Poly
+record ego.lane.polygon as egoLanePoly
+record adv1.lane.polygon as adv1LanePoly
+record adv2.lane.polygon as adv2LanePoly
+record adv3.lane.polygon as adv3LanePoly
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left.sgraph b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left.sgraph
new file mode 100644
index 0000000..eb19a9a
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left.sgraph
@@ -0,0 +1,23 @@
+# ID 0
+# Node list
+0 off rule0 monitor
+1 on rule1 monitor
+2 off rule2 monitor
+3 off rule3 monitor
+4 off rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 on rule8 monitor
+# Edge list
+0 3
+1 3
+2 3
+3 4
+3 5
+4 7
+4 8
+5 7
+5 8
+7 6
+8 6
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_00.graph b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_00.graph
new file mode 100644
index 0000000..a43073c
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_00.graph
@@ -0,0 +1,16 @@
+# ID 0
+# Node list
+0 off rule0 monitor
+1 on rule1 monitor
+2 off rule2 monitor
+3 off rule3 monitor
+4 off rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 on rule8 monitor
+# Edge list
+0 3
+3 4
+4 7
+7 6
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_01.graph b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_01.graph
new file mode 100644
index 0000000..e05f098
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_01.graph
@@ -0,0 +1,16 @@
+# ID 1
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+2 on rule2 monitor
+3 on rule3 monitor
+4 on rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 off rule8 monitor
+# Edge list
+0 3
+1 3
+2 3
+3 8
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_02.graph b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_02.graph
new file mode 100644
index 0000000..5c890ba
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_02.graph
@@ -0,0 +1,15 @@
+# ID 2
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+2 on rule2 monitor
+3 on rule3 monitor
+4 on rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 off rule8 monitor
+# Edge list
+2 3
+3 5
+5 6
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_rulebook.py b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_rulebook.py
new file mode 100644
index 0000000..6c8dbc1
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_rulebook.py
@@ -0,0 +1,58 @@
+import numpy as np
+
+from verifai.rulebook import rulebook
+
+class rulebook_multileft(rulebook):
+ iteration = 0
+
+ def __init__(self, graph_path, rule_file, save_path=None, single_graph=False, using_sampler=-1, exploration_ratio=2.0):
+ rulebook.using_sampler = using_sampler
+ rulebook.exploration_ratio = exploration_ratio
+ super().__init__(graph_path, rule_file, single_graph=single_graph)
+ self.save_path = save_path
+
+ def evaluate(self, simulation):
+ # Extract trajectory information
+ positions = np.array(simulation.result.trajectory)
+ ego_dist_to_intersection = np.array(simulation.result.records["egoDistToIntersection"])
+
+ # Find switching points, i.e., ego has reached the intersection / ego has finished the left turn
+ switch_idx_1 = len(simulation.result.trajectory)
+ switch_idx_2 = len(simulation.result.trajectory)
+ for i in range(len(ego_dist_to_intersection)):
+ if ego_dist_to_intersection[i][1] == 0 and switch_idx_1 == len(simulation.result.trajectory):
+ switch_idx_1 = i
+ break
+ if switch_idx_1 < len(simulation.result.trajectory):
+ for i in reversed(range(switch_idx_1, len(ego_dist_to_intersection))):
+ if ego_dist_to_intersection[i][1] == 0:
+ switch_idx_2 = i + 1
+ break
+ assert switch_idx_1 <= switch_idx_2
+
+ # Evaluation
+ indices_0 = np.arange(0, switch_idx_1)
+ indices_1 = np.arange(switch_idx_1, switch_idx_2)
+ indices_2 = np.arange(switch_idx_2, len(simulation.result.trajectory))
+ #print('Indices:', indices_0, indices_1, indices_2)
+ if self.single_graph:
+ rho0 = self.evaluate_segment(simulation, 0, indices_0)
+ rho1 = self.evaluate_segment(simulation, 0, indices_1)
+ rho2 = self.evaluate_segment(simulation, 0, indices_2)
+ print('Actual rho:')
+ for r in rho0:
+ print(r, end=' ')
+ print()
+ for r in rho1:
+ print(r, end=' ')
+ print()
+ for r in rho2:
+ print(r, end=' ')
+ print()
+ rho = self.evaluate_segment(simulation, 0, np.arange(0, len(simulation.result.trajectory)))
+ return np.array([rho])
+ rho0 = self.evaluate_segment(simulation, 0, indices_0)
+ rho1 = self.evaluate_segment(simulation, 1, indices_1)
+ rho2 = self.evaluate_segment(simulation, 2, indices_2)
+ return np.array([rho0, rho1, rho2])
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_spec.py b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_spec.py
new file mode 100644
index 0000000..25680d5
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2left/multi_verifai2left_spec.py
@@ -0,0 +1,74 @@
+import numpy as np
+
+def rule0(simulation, indices): # B, 1: safe distance to adv1
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [1], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule1(simulation, indices): # B, 2: safe distance to adv2
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [2], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule2(simulation, indices): # B, 3: safe distance to adv3
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [3], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule3(simulation, indices): # C: stay in drivable area
+ if indices.size == 0:
+ return 1
+ distance_to_drivable = np.array(simulation.result.records["egoDistToDrivableRegion"])
+ rho = -np.max(distance_to_drivable[indices], axis=0)[1]
+ return rho
+
+def rule4(simulation, indices): # D, 1: stay in the correct side of the road, before intersection
+ if indices.size == 0:
+ return 1
+ distance_to_lane_group = np.array(simulation.result.records["egoDistToEgoInitLane"])
+ rho = -np.max(distance_to_lane_group[indices], axis=0)[1]
+ return rho
+
+def rule5(simulation, indices): # D, 2: stay in the correct side of the road, after intersection
+ if indices.size == 0:
+ return 1
+ distance_to_lane_group = np.array(simulation.result.records["egoDistToEgoEndLane"])
+ rho = -np.max(distance_to_lane_group[indices], axis=0)[1]
+ return rho
+
+def rule6(simulation, indices): # F: lane keeping
+ if indices.size == 0:
+ return 1
+ distance_to_lane_center = np.array(simulation.result.records["egoDistToEgoLaneCenterline"])
+ rho = 0.4 - np.max(distance_to_lane_center[indices], axis=0)[1]
+ return rho
+
+def rule7(simulation, indices): # H, 1: reach intersection
+ if indices.size == 0:
+ return 1
+ if max(indices) < len(simulation.result.trajectory) - 1:
+ return 1
+ ego_dist_to_intersection = np.array(simulation.result.records["egoDistToIntersection"])
+ rho = -np.min(ego_dist_to_intersection[indices], axis=0)[1]
+ return rho
+
+def rule8(simulation, indices): # H, 2: reach end lane
+ if indices.size == 0:
+ return 1
+ if max(indices) < len(simulation.result.trajectory) - 1:
+ return 1
+ ego_dist_to_end_lane = np.array(simulation.result.records["egoDistToEgoEndLane"])
+ rho = -np.min(ego_dist_to_end_lane[indices], axis=0)[1]
+ return rho
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2left/util/multi_verifai2left_analyze_diversity.py b/examples/dynamic_rulebook/multi_verifai2left/util/multi_verifai2left_analyze_diversity.py
new file mode 100644
index 0000000..a721de5
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2left/util/multi_verifai2left_analyze_diversity.py
@@ -0,0 +1,48 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+directory = sys.argv[1]
+all_files = os.listdir(directory)
+all_files = [f for f in all_files if f.endswith('.csv') and f.startswith(sys.argv[2]+'.')]
+mode = sys.argv[3] # multi / single
+
+fig = plt.figure()
+ax = fig.add_subplot(projection='3d')
+count = 0
+ego_speed = []
+ego_brake = []
+adv_speed = []
+adv1_dist = []
+for file in all_files:
+ infile = open(directory+'/'+file, 'r')
+ lines = infile.readlines()
+ if mode == 'single':
+ for i in range(1, len(lines)):
+ line = lines[i] #TODO: identify the counterexamples
+ ego_speed.append(float(line.split(',')[-10]))
+ ego_brake.append(float(line.split(',')[-11]))
+ adv_speed.append(float(line.split(',')[-12]))
+ adv1_dist.append(float(line.split(',')[-13]))
+ else:
+ for i in range(1, len(lines), 3):
+ line1 = lines[i]
+ line2 = lines[i+1]
+ line3 = lines[i+2] #TODO: identify the counterexamples
+ ego_speed.append(float(line1.split(',')[-10]))
+ ego_brake.append(float(line1.split(',')[-11]))
+ adv_speed.append(float(line1.split(',')[-12]))
+ adv1_dist.append(float(line1.split(',')[-13]))
+
+ax.scatter(ego_speed, adv_speed, adv1_dist)
+ax.set_xlabel('EGO_SPEED')
+ax.set_ylabel('ADV_SPEED')
+ax.set_zlabel('ADV1_DIST')
+plt.savefig(directory+'/'+sys.argv[2]+'_scatter.png')
+
+print("Standard deviation of ego_speed:", np.std(ego_speed), len(ego_speed))
+print("Standard deviation of adv_speed:", np.std(adv_speed), len(adv_speed))
+print("Standard deviation of ego_brake:", np.std(ego_brake), len(ego_brake))
+print("Standard deviation of adv1_dist:", np.std(adv1_dist), len(adv1_dist))
+print()
diff --git a/examples/dynamic_rulebook/multi_verifai2left/util/multi_verifai2left_collect_result.py b/examples/dynamic_rulebook/multi_verifai2left/util/multi_verifai2left_collect_result.py
new file mode 100644
index 0000000..2fed830
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2left/util/multi_verifai2left_collect_result.py
@@ -0,0 +1,144 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import itertools
+
+infile = open(sys.argv[1], 'r') # *.txt
+mode = sys.argv[2] # multi / single
+order = sys.argv[3] # alternate / sequential
+
+# error weights
+result_count_0 = [[] for i in range(3)]
+result_count_1 = [[] for i in range(3)]
+result_count_2 = [[] for i in range(3)]
+# counterexample types
+counterexample_type_0 = [{} for i in range(3)]
+counterexample_type_1 = [{} for i in range(3)]
+counterexample_type_2 = [{} for i in range(3)]
+curr_source = 0
+lines = infile.readlines()
+infile.close()
+
+for i in range(len(lines)):
+ if mode == 'multi':
+ if 'RHO' in lines[i]:
+ line = lines[i+1].strip().split(' ')
+ val1 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val1.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val1) == 9, 'Invalid length of rho'
+ result_count_0[curr_source].append(val1[0]*16 + val1[3]*8 + val1[4]*4 + val1[7]*2 + val1[6]*1)
+ if tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]])) in counterexample_type_0[curr_source]:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]]))] += 1
+ else:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]]))] = 1
+
+ line = lines[i+2].strip().split(' ')
+ val2 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val2.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val2) == 9, 'Invalid length of rho'
+ result_count_1[curr_source].append(val2[0]*4 + val2[1]*4 + val2[2]*4 + val2[3]*2 + val2[8]*1)
+ if tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]])) in counterexample_type_1[curr_source]:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]]))] += 1
+ else:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]]))] = 1
+
+ line = lines[i+3].strip().split(' ')
+ val3 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val3.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val3) == 9, 'Invalid length of rho'
+ result_count_2[curr_source].append(val3[2]*8 + val3[3]*4 + val3[5]*2 + val3[6]*1)
+ if tuple(1*np.array([val3[2], val3[3], val3[5], val3[6]])) in counterexample_type_2[curr_source]:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[2], val3[3], val3[5], val3[6]]))] += 1
+ else:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[2], val3[3], val3[5], val3[6]]))] = 1
+
+ if order == '-1':
+ curr_source = curr_source + 1 if curr_source < 2 else 0
+ else:
+ if 'Actual rho' in lines[i]:
+ line = lines[i+1].strip().split(' ')
+ val1 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val1.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val1) == 9, 'Invalid length of rho'
+ result_count_0[curr_source].append(val1[0]*16 + val1[3]*8 + val1[4]*4 + val1[7]*2 + val1[6]*1)
+ if tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]])) in counterexample_type_0[curr_source]:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]]))] += 1
+ else:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]]))] = 1
+
+ line = lines[i+2].strip().split(' ')
+ val2 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val2.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val2) == 9, 'Invalid length of rho'
+ result_count_1[curr_source].append(val2[0]*4 + val2[1]*4 + val2[2]*4 + val2[3]*2 + val2[8]*1)
+ if tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]])) in counterexample_type_1[curr_source]:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]]))] += 1
+ else:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]]))] = 1
+
+ line = lines[i+3].strip().split(' ')
+ val3 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val3.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val3) == 9, 'Invalid length of rho'
+ result_count_2[curr_source].append(val3[2]*8 + val3[3]*4 + val3[5]*2 + val3[6]*1)
+ if tuple(1*np.array([val3[2], val3[3], val3[5], val3[6]])) in counterexample_type_2[curr_source]:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[2], val3[3], val3[5], val3[6]]))] += 1
+ else:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[2], val3[3], val3[5], val3[6]]))] = 1
+
+ if order == '-1':
+ curr_source = curr_source + 1 if curr_source < 2 else 0
+
+print('Error weights')
+print('segment 0:')
+for i in range(1):
+ print('average:', np.mean(result_count_0[i]), 'max:', np.max(result_count_0[i]), 'percentage:', float(np.count_nonzero(result_count_0[i])/len(result_count_0[i])), result_count_0[i])
+print('segment 1:')
+for i in range(1):
+ print('average:', np.mean(result_count_1[i]), 'max:', np.max(result_count_1[i]), 'percentage:', float(np.count_nonzero(result_count_1[i])/len(result_count_1[i])), result_count_1[i])
+print('segment 2:')
+for i in range(1):
+ print('average:', np.mean(result_count_2[i]), 'max:', np.max(result_count_2[i]), 'percentage:', float(np.count_nonzero(result_count_2[i])/len(result_count_2[i])), result_count_2[i])
+
+print('\nCounterexample types')
+print('segment 0:')
+for i in range(1):
+ print('Types:', len(counterexample_type_0[i]))
+ for key, value in reversed(sorted(counterexample_type_0[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print('segment 1:')
+for i in range(1):
+ print('Types:', len(counterexample_type_1[i]))
+ for key, value in reversed(sorted(counterexample_type_1[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print('segment 2:')
+for i in range(1):
+ print('Types:', len(counterexample_type_2[i]))
+ for key, value in reversed(sorted(counterexample_type_2[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print()
diff --git a/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right.py b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right.py
new file mode 100644
index 0000000..46c0dd1
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right.py
@@ -0,0 +1,51 @@
+import sys
+import os
+sys.path.append(os.path.abspath("."))
+import random
+import numpy as np
+
+from multi import *
+from multi_verifai2right_rulebook import rulebook_multiright
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--scenic-path', '-sp', type=str, default='uberCrashNewton.scenic',
+ help='Path to Scenic script')
+ parser.add_argument('--graph-path', '-gp', type=str, default=None,
+ help='Path to graph file')
+ parser.add_argument('--rule-path', '-rp', type=str, default=None,
+ help='Path to rule file')
+ parser.add_argument('--output-dir', '-o', type=str, default=None,
+ help='Directory to save output trajectories')
+ parser.add_argument('--output-csv-dir', '-co', type=str, default=None,
+ help='Directory to save output error tables (csv files)')
+ parser.add_argument('--parallel', action='store_true')
+ parser.add_argument('--num-workers', type=int, default=5, help='Number of parallel workers')
+ parser.add_argument('--sampler-type', '-s', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--experiment-name', '-e', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--model', '-m', type=str, default='scenic.simulators.newtonian.driving_model')
+ parser.add_argument('--headless', action='store_true')
+ parser.add_argument('--n-iters', '-n', type=int, default=None, help='Number of simulations to run')
+ parser.add_argument('--max-time', type=int, default=None, help='Maximum amount of time to run simulations')
+ parser.add_argument('--single-graph', action='store_true', help='Only a unified priority graph')
+ parser.add_argument('--seed', type=int, default=0, help='Random seed')
+ parser.add_argument('--using-sampler', type=int, default=-1, help='Assigning sampler to use')
+ parser.add_argument('--max-simulation-steps', type=int, default=300, help='Maximum number of simulation steps')
+ parser.add_argument('--exploration-ratio', type=float, default=2.0, help='Exploration ratio')
+ args = parser.parse_args()
+ if args.n_iters is None and args.max_time is None:
+ raise ValueError('At least one of --n-iters or --max-time must be set')
+
+ random.seed(args.seed)
+ np.random.seed(args.seed)
+
+ rb = rulebook_multiright(args.graph_path, args.rule_path, save_path=args.output_dir, single_graph=args.single_graph,
+ using_sampler=args.using_sampler, exploration_ratio=args.exploration_ratio)
+ run_experiments(args.scenic_path, rulebook=rb,
+ parallel=args.parallel, model=args.model,
+ sampler_type=args.sampler_type, headless=args.headless,
+ num_workers=args.num_workers, output_dir=args.output_csv_dir, experiment_name=args.experiment_name,
+ max_time=args.max_time, n_iters=args.n_iters, max_steps=args.max_simulation_steps)
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right.scenic b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right.scenic
new file mode 100644
index 0000000..58839b9
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right.scenic
@@ -0,0 +1,137 @@
+"""
+TITLE: Verifai 2.0 Right Turn
+AUTHOR: Kai-Chun Chang, kaichunchang@berkeley.edu
+"""
+
+#################################
+# MAP AND MODEL #
+#################################
+
+param map = localPath('../maps/Town05.xodr')
+param carla_map = 'Town05'
+model scenic.domains.driving.model
+
+#################################
+# CONSTANTS #
+#################################
+
+MODEL = 'vehicle.lincoln.mkz_2017' #'vehicle.toyota.prius'
+MODEL_ADV = 'vehicle.lincoln.mkz_2017'
+
+EGO_INIT_DIST = [30, 40]
+param EGO_SPEED = VerifaiRange(7, 10)
+param EGO_BRAKE = VerifaiRange(0.8, 1.0)
+
+param ADV1_DIST = VerifaiRange(6, 10)
+ADV_INIT_DIST = [15, 25]
+param ADV_SPEED = VerifaiRange(5, 8)
+
+PED_MIN_SPEED = 1.0
+PED_THRESHOLD = 20
+PED_FINAL_SPEED = 1.0
+
+SAFETY_DIST = 8
+CRASH_DIST = 5
+TERM_DIST = 80
+
+#################################
+# AGENT BEHAVIORS #
+#################################
+
+behavior EgoBehavior(trajectory):
+ try:
+ do FollowTrajectoryBehavior(target_speed=globalParameters.EGO_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.EGO_SPEED)
+ interrupt when withinDistanceToAnyObjs(self, SAFETY_DIST):
+ take SetBrakeAction(globalParameters.EGO_BRAKE)
+
+behavior Adv1Behavior(trajectory):
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+
+behavior Adv2Behavior(trajectory):
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+
+behavior Adv3Behavior(trajectory):
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+
+#################################
+# SPATIAL RELATIONS #
+#################################
+
+intersection = Uniform(*filter(lambda i: i.is4Way, network.intersections))
+
+# ego: right turn from S to E
+egoManeuver = Uniform(*filter(lambda m: m.type is ManeuverType.RIGHT_TURN, intersection.maneuvers))
+egoInitLane = egoManeuver.startLane
+egoTrajectory = [egoInitLane, egoManeuver.connectingLane, egoManeuver.endLane]
+egoSpawnPt = new OrientedPoint in egoInitLane.centerline
+
+# adv1: straight from S to N
+adv1InitLane = egoInitLane
+adv1Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv1InitLane.maneuvers))
+adv1Trajectory = [adv1InitLane, adv1Maneuver.connectingLane, adv1Maneuver.endLane]
+
+# adv2: straight from W to E
+adv2InitLane = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT,
+ Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, egoInitLane.maneuvers)).conflictingManeuvers)).startLane
+adv2Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv2InitLane.maneuvers))
+adv2Trajectory = [adv2InitLane, adv2Maneuver.connectingLane, adv2Maneuver.endLane]
+adv2SpawnPt = new OrientedPoint in adv2InitLane.centerline
+
+# adv3: straight from E to W
+adv3InitLane = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv2Maneuver.reverseManeuvers)).startLane
+adv3Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv3InitLane.maneuvers))
+adv3Trajectory = [adv3InitLane, adv3Maneuver.connectingLane, adv3Maneuver.endLane]
+adv3SpawnPt = new OrientedPoint in adv3InitLane.centerline
+
+#################################
+# SCENARIO SPECIFICATION #
+#################################
+
+ego = new Car at egoSpawnPt,
+ with blueprint MODEL,
+ with behavior EgoBehavior(egoTrajectory)
+
+adv1 = new Car following roadDirection for globalParameters.ADV1_DIST,
+ with blueprint MODEL_ADV,
+ with behavior Adv1Behavior(adv1Trajectory)
+
+adv2 = new Car at adv2SpawnPt,
+ with blueprint MODEL_ADV,
+ with behavior Adv2Behavior(adv2Trajectory)
+
+adv3 = new Car at adv3SpawnPt,
+ with blueprint MODEL_ADV,
+ with behavior Adv3Behavior(adv3Trajectory)
+
+require EGO_INIT_DIST[0] <= (distance to intersection) <= EGO_INIT_DIST[1]
+require ADV_INIT_DIST[0] <= (distance from adv2 to intersection) <= ADV_INIT_DIST[1]
+require ADV_INIT_DIST[0] <= (distance from adv3 to intersection) <= ADV_INIT_DIST[1]
+require adv3InitLane.road is egoManeuver.endLane.road
+terminate when (distance to egoSpawnPt) > TERM_DIST
+
+#################################
+# RECORDING #
+#################################
+
+record (ego in network.drivableRegion) as egoIsInDrivableRegion
+record (distance from ego to network.drivableRegion) as egoDistToDrivableRegion
+record (distance from ego to egoInitLane.group) as egoDistToEgoInitLane
+record (distance from ego to egoManeuver.endLane.group) as egoDistToEgoEndLane
+record (distance from ego to ego.lane.centerline) as egoDistToEgoLaneCenterline
+record (distance from ego to intersection) as egoDistToIntersection
+
+record (distance from ego to adv1) as egoDistToAdv1
+record (distance to egoSpawnPt) as egoDistToEgoSpawnPt
+
+record ego._boundingPolygon as egoPoly
+record adv1._boundingPolygon as adv1Poly
+record adv2._boundingPolygon as adv2Poly
+record adv3._boundingPolygon as adv3Poly
+record ego.lane.polygon as egoLanePoly
+record adv1.lane.polygon as adv1LanePoly
+record adv2.lane.polygon as adv2LanePoly
+record adv3.lane.polygon as adv3LanePoly
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right.sgraph b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right.sgraph
new file mode 100644
index 0000000..eb19a9a
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right.sgraph
@@ -0,0 +1,23 @@
+# ID 0
+# Node list
+0 off rule0 monitor
+1 on rule1 monitor
+2 off rule2 monitor
+3 off rule3 monitor
+4 off rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 on rule8 monitor
+# Edge list
+0 3
+1 3
+2 3
+3 4
+3 5
+4 7
+4 8
+5 7
+5 8
+7 6
+8 6
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_00.graph b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_00.graph
new file mode 100644
index 0000000..a43073c
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_00.graph
@@ -0,0 +1,16 @@
+# ID 0
+# Node list
+0 off rule0 monitor
+1 on rule1 monitor
+2 off rule2 monitor
+3 off rule3 monitor
+4 off rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 on rule8 monitor
+# Edge list
+0 3
+3 4
+4 7
+7 6
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_01.graph b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_01.graph
new file mode 100644
index 0000000..e05f098
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_01.graph
@@ -0,0 +1,16 @@
+# ID 1
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+2 on rule2 monitor
+3 on rule3 monitor
+4 on rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 off rule8 monitor
+# Edge list
+0 3
+1 3
+2 3
+3 8
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_02.graph b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_02.graph
new file mode 100644
index 0000000..034e93e
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_02.graph
@@ -0,0 +1,15 @@
+# ID 2
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+2 on rule2 monitor
+3 on rule3 monitor
+4 on rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 off rule8 monitor
+# Edge list
+1 3
+3 5
+5 6
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_rulebook.py b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_rulebook.py
new file mode 100644
index 0000000..d7443b1
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_rulebook.py
@@ -0,0 +1,58 @@
+import numpy as np
+
+from verifai.rulebook import rulebook
+
+class rulebook_multiright(rulebook):
+ iteration = 0
+
+ def __init__(self, graph_path, rule_file, save_path=None, single_graph=False, using_sampler=-1, exploration_ratio=2.0):
+ rulebook.using_sampler = using_sampler
+ rulebook.exploration_ratio = exploration_ratio
+ super().__init__(graph_path, rule_file, single_graph=single_graph)
+ self.save_path = save_path
+
+ def evaluate(self, simulation):
+ # Extract trajectory information
+ positions = np.array(simulation.result.trajectory)
+ ego_dist_to_intersection = np.array(simulation.result.records["egoDistToIntersection"])
+
+ # Find switching points, i.e., ego has reached the intersection / ego has finished the right turn
+ switch_idx_1 = len(simulation.result.trajectory)
+ switch_idx_2 = len(simulation.result.trajectory)
+ for i in range(len(ego_dist_to_intersection)):
+ if ego_dist_to_intersection[i][1] == 0 and switch_idx_1 == len(simulation.result.trajectory):
+ switch_idx_1 = i
+ break
+ if switch_idx_1 < len(simulation.result.trajectory):
+ for i in reversed(range(switch_idx_1, len(ego_dist_to_intersection))):
+ if ego_dist_to_intersection[i][1] == 0:
+ switch_idx_2 = i + 1
+ break
+ assert switch_idx_1 <= switch_idx_2
+
+ # Evaluation
+ indices_0 = np.arange(0, switch_idx_1)
+ indices_1 = np.arange(switch_idx_1, switch_idx_2)
+ indices_2 = np.arange(switch_idx_2, len(simulation.result.trajectory))
+ #print('Indices:', indices_0, indices_1, indices_2)
+ if self.single_graph:
+ rho0 = self.evaluate_segment(simulation, 0, indices_0)
+ rho1 = self.evaluate_segment(simulation, 0, indices_1)
+ rho2 = self.evaluate_segment(simulation, 0, indices_2)
+ print('Actual rho:')
+ for r in rho0:
+ print(r, end=' ')
+ print()
+ for r in rho1:
+ print(r, end=' ')
+ print()
+ for r in rho2:
+ print(r, end=' ')
+ print()
+ rho = self.evaluate_segment(simulation, 0, np.arange(0, len(simulation.result.trajectory)))
+ return np.array([rho])
+ rho0 = self.evaluate_segment(simulation, 0, indices_0)
+ rho1 = self.evaluate_segment(simulation, 1, indices_1)
+ rho2 = self.evaluate_segment(simulation, 2, indices_2)
+ return np.array([rho0, rho1, rho2])
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_spec.py b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_spec.py
new file mode 100644
index 0000000..25680d5
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2right/multi_verifai2right_spec.py
@@ -0,0 +1,74 @@
+import numpy as np
+
+def rule0(simulation, indices): # B, 1: safe distance to adv1
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [1], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule1(simulation, indices): # B, 2: safe distance to adv2
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [2], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule2(simulation, indices): # B, 3: safe distance to adv3
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [3], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule3(simulation, indices): # C: stay in drivable area
+ if indices.size == 0:
+ return 1
+ distance_to_drivable = np.array(simulation.result.records["egoDistToDrivableRegion"])
+ rho = -np.max(distance_to_drivable[indices], axis=0)[1]
+ return rho
+
+def rule4(simulation, indices): # D, 1: stay in the correct side of the road, before intersection
+ if indices.size == 0:
+ return 1
+ distance_to_lane_group = np.array(simulation.result.records["egoDistToEgoInitLane"])
+ rho = -np.max(distance_to_lane_group[indices], axis=0)[1]
+ return rho
+
+def rule5(simulation, indices): # D, 2: stay in the correct side of the road, after intersection
+ if indices.size == 0:
+ return 1
+ distance_to_lane_group = np.array(simulation.result.records["egoDistToEgoEndLane"])
+ rho = -np.max(distance_to_lane_group[indices], axis=0)[1]
+ return rho
+
+def rule6(simulation, indices): # F: lane keeping
+ if indices.size == 0:
+ return 1
+ distance_to_lane_center = np.array(simulation.result.records["egoDistToEgoLaneCenterline"])
+ rho = 0.4 - np.max(distance_to_lane_center[indices], axis=0)[1]
+ return rho
+
+def rule7(simulation, indices): # H, 1: reach intersection
+ if indices.size == 0:
+ return 1
+ if max(indices) < len(simulation.result.trajectory) - 1:
+ return 1
+ ego_dist_to_intersection = np.array(simulation.result.records["egoDistToIntersection"])
+ rho = -np.min(ego_dist_to_intersection[indices], axis=0)[1]
+ return rho
+
+def rule8(simulation, indices): # H, 2: reach end lane
+ if indices.size == 0:
+ return 1
+ if max(indices) < len(simulation.result.trajectory) - 1:
+ return 1
+ ego_dist_to_end_lane = np.array(simulation.result.records["egoDistToEgoEndLane"])
+ rho = -np.min(ego_dist_to_end_lane[indices], axis=0)[1]
+ return rho
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2right/util/multi_verifai2right_analyze_diversity.py b/examples/dynamic_rulebook/multi_verifai2right/util/multi_verifai2right_analyze_diversity.py
new file mode 100644
index 0000000..a721de5
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2right/util/multi_verifai2right_analyze_diversity.py
@@ -0,0 +1,48 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+directory = sys.argv[1]
+all_files = os.listdir(directory)
+all_files = [f for f in all_files if f.endswith('.csv') and f.startswith(sys.argv[2]+'.')]
+mode = sys.argv[3] # multi / single
+
+fig = plt.figure()
+ax = fig.add_subplot(projection='3d')
+count = 0
+ego_speed = []
+ego_brake = []
+adv_speed = []
+adv1_dist = []
+for file in all_files:
+ infile = open(directory+'/'+file, 'r')
+ lines = infile.readlines()
+ if mode == 'single':
+ for i in range(1, len(lines)):
+ line = lines[i] #TODO: identify the counterexamples
+ ego_speed.append(float(line.split(',')[-10]))
+ ego_brake.append(float(line.split(',')[-11]))
+ adv_speed.append(float(line.split(',')[-12]))
+ adv1_dist.append(float(line.split(',')[-13]))
+ else:
+ for i in range(1, len(lines), 3):
+ line1 = lines[i]
+ line2 = lines[i+1]
+ line3 = lines[i+2] #TODO: identify the counterexamples
+ ego_speed.append(float(line1.split(',')[-10]))
+ ego_brake.append(float(line1.split(',')[-11]))
+ adv_speed.append(float(line1.split(',')[-12]))
+ adv1_dist.append(float(line1.split(',')[-13]))
+
+ax.scatter(ego_speed, adv_speed, adv1_dist)
+ax.set_xlabel('EGO_SPEED')
+ax.set_ylabel('ADV_SPEED')
+ax.set_zlabel('ADV1_DIST')
+plt.savefig(directory+'/'+sys.argv[2]+'_scatter.png')
+
+print("Standard deviation of ego_speed:", np.std(ego_speed), len(ego_speed))
+print("Standard deviation of adv_speed:", np.std(adv_speed), len(adv_speed))
+print("Standard deviation of ego_brake:", np.std(ego_brake), len(ego_brake))
+print("Standard deviation of adv1_dist:", np.std(adv1_dist), len(adv1_dist))
+print()
diff --git a/examples/dynamic_rulebook/multi_verifai2right/util/multi_verifai2right_collect_result.py b/examples/dynamic_rulebook/multi_verifai2right/util/multi_verifai2right_collect_result.py
new file mode 100644
index 0000000..3484b7f
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2right/util/multi_verifai2right_collect_result.py
@@ -0,0 +1,144 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import itertools
+
+infile = open(sys.argv[1], 'r') # *.txt
+mode = sys.argv[2] # multi / single
+order = sys.argv[3] # alternate / sequential
+
+# error weights
+result_count_0 = [[] for i in range(3)]
+result_count_1 = [[] for i in range(3)]
+result_count_2 = [[] for i in range(3)]
+# counterexample types
+counterexample_type_0 = [{} for i in range(3)]
+counterexample_type_1 = [{} for i in range(3)]
+counterexample_type_2 = [{} for i in range(3)]
+curr_source = 0
+lines = infile.readlines()
+infile.close()
+
+for i in range(len(lines)):
+ if mode == 'multi':
+ if 'RHO' in lines[i]:
+ line = lines[i+1].strip().split(' ')
+ val1 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val1.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val1) == 9, 'Invalid length of rho'
+ result_count_0[curr_source].append(val1[0]*16 + val1[3]*8 + val1[4]*4 + val1[7]*2 + val1[6]*1)
+ if tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]])) in counterexample_type_0[curr_source]:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]]))] += 1
+ else:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]]))] = 1
+
+ line = lines[i+2].strip().split(' ')
+ val2 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val2.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val2) == 9, 'Invalid length of rho'
+ result_count_1[curr_source].append(val2[0]*4 + val2[1]*4 + val2[2]*4 + val2[3]*2 + val2[8]*1)
+ if tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]])) in counterexample_type_1[curr_source]:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]]))] += 1
+ else:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]]))] = 1
+
+ line = lines[i+3].strip().split(' ')
+ val3 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val3.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val3) == 9, 'Invalid length of rho'
+ result_count_2[curr_source].append(val3[1]*8 + val3[3]*4 + val3[5]*2 + val3[6]*1)
+ if tuple(1*np.array([val3[1], val3[3], val3[5], val3[6]])) in counterexample_type_2[curr_source]:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[1], val3[3], val3[5], val3[6]]))] += 1
+ else:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[1], val3[3], val3[5], val3[6]]))] = 1
+
+ if order == '-1':
+ curr_source = curr_source + 1 if curr_source < 2 else 0
+ else:
+ if 'Actual rho' in lines[i]:
+ line = lines[i+1].strip().split(' ')
+ val1 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val1.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val1) == 9, 'Invalid length of rho'
+ result_count_0[curr_source].append(val1[0]*16 + val1[3]*8 + val1[4]*4 + val1[7]*2 + val1[6]*1)
+ if tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]])) in counterexample_type_0[curr_source]:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]]))] += 1
+ else:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]]))] = 1
+
+ line = lines[i+2].strip().split(' ')
+ val2 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val2.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val2) == 9, 'Invalid length of rho'
+ result_count_1[curr_source].append(val2[0]*4 + val2[1]*4 + val2[2]*4 + val2[3]*2 + val2[8]*1)
+ if tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]])) in counterexample_type_1[curr_source]:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]]))] += 1
+ else:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]]))] = 1
+
+ line = lines[i+3].strip().split(' ')
+ val3 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val3.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val3) == 9, 'Invalid length of rho'
+ result_count_2[curr_source].append(val3[1]*8 + val3[3]*4 + val3[5]*2 + val3[6]*1)
+ if tuple(1*np.array([val3[1], val3[3], val3[5], val3[6]])) in counterexample_type_2[curr_source]:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[1], val3[3], val3[5], val3[6]]))] += 1
+ else:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[1], val3[3], val3[5], val3[6]]))] = 1
+
+ if order == '-1':
+ curr_source = curr_source + 1 if curr_source < 2 else 0
+
+print('Error weights')
+print('segment 0:')
+for i in range(1):
+ print('average:', np.mean(result_count_0[i]), 'max:', np.max(result_count_0[i]), 'percentage:', float(np.count_nonzero(result_count_0[i])/len(result_count_0[i])), result_count_0[i])
+print('segment 1:')
+for i in range(1):
+ print('average:', np.mean(result_count_1[i]), 'max:', np.max(result_count_1[i]), 'percentage:', float(np.count_nonzero(result_count_1[i])/len(result_count_1[i])), result_count_1[i])
+print('segment 2:')
+for i in range(1):
+ print('average:', np.mean(result_count_2[i]), 'max:', np.max(result_count_2[i]), 'percentage:', float(np.count_nonzero(result_count_2[i])/len(result_count_2[i])), result_count_2[i])
+
+print('\nCounterexample types')
+print('segment 0:')
+for i in range(1):
+ print('Types:', len(counterexample_type_0[i]))
+ for key, value in reversed(sorted(counterexample_type_0[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print('segment 1:')
+for i in range(1):
+ print('Types:', len(counterexample_type_1[i]))
+ for key, value in reversed(sorted(counterexample_type_1[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print('segment 2:')
+for i in range(1):
+ print('Types:', len(counterexample_type_2[i]))
+ for key, value in reversed(sorted(counterexample_type_2[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print()
diff --git a/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight.py b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight.py
new file mode 100644
index 0000000..a669299
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight.py
@@ -0,0 +1,51 @@
+import sys
+import os
+sys.path.append(os.path.abspath("."))
+import random
+import numpy as np
+
+from multi import *
+from multi_verifai2straight_rulebook import rulebook_multistraight
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--scenic-path', '-sp', type=str, default='uberCrashNewton.scenic',
+ help='Path to Scenic script')
+ parser.add_argument('--graph-path', '-gp', type=str, default=None,
+ help='Path to graph file')
+ parser.add_argument('--rule-path', '-rp', type=str, default=None,
+ help='Path to rule file')
+ parser.add_argument('--output-dir', '-o', type=str, default=None,
+ help='Directory to save output trajectories')
+ parser.add_argument('--output-csv-dir', '-co', type=str, default=None,
+ help='Directory to save output error tables (csv files)')
+ parser.add_argument('--parallel', action='store_true')
+ parser.add_argument('--num-workers', type=int, default=5, help='Number of parallel workers')
+ parser.add_argument('--sampler-type', '-s', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--experiment-name', '-e', type=str, default=None,
+ help='verifaiSamplerType to use')
+ parser.add_argument('--model', '-m', type=str, default='scenic.simulators.newtonian.driving_model')
+ parser.add_argument('--headless', action='store_true')
+ parser.add_argument('--n-iters', '-n', type=int, default=None, help='Number of simulations to run')
+ parser.add_argument('--max-time', type=int, default=None, help='Maximum amount of time to run simulations')
+ parser.add_argument('--single-graph', action='store_true', help='Only a unified priority graph')
+ parser.add_argument('--seed', type=int, default=0, help='Random seed')
+ parser.add_argument('--using-sampler', type=int, default=-1, help='Assigning sampler to use')
+ parser.add_argument('--max-simulation-steps', type=int, default=300, help='Maximum number of simulation steps')
+ parser.add_argument('--exploration-ratio', type=float, default=2.0, help='Exploration ratio')
+ args = parser.parse_args()
+ if args.n_iters is None and args.max_time is None:
+ raise ValueError('At least one of --n-iters or --max-time must be set')
+
+ random.seed(args.seed)
+ np.random.seed(args.seed)
+
+ rb = rulebook_multistraight(args.graph_path, args.rule_path, save_path=args.output_dir, single_graph=args.single_graph,
+ using_sampler=args.using_sampler, exploration_ratio=args.exploration_ratio)
+ run_experiments(args.scenic_path, rulebook=rb,
+ parallel=args.parallel, model=args.model,
+ sampler_type=args.sampler_type, headless=args.headless,
+ num_workers=args.num_workers, output_dir=args.output_csv_dir, experiment_name=args.experiment_name,
+ max_time=args.max_time, n_iters=args.n_iters, max_steps=args.max_simulation_steps)
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight.scenic b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight.scenic
new file mode 100644
index 0000000..cd39edb
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight.scenic
@@ -0,0 +1,136 @@
+"""
+TITLE: Verifai 2.0 Going Straight
+AUTHOR: Kai-Chun Chang, kaichunchang@berkeley.edu
+"""
+
+#################################
+# MAP AND MODEL #
+#################################
+
+param map = localPath('../maps/Town05.xodr')
+param carla_map = 'Town05'
+model scenic.domains.driving.model
+
+#################################
+# CONSTANTS #
+#################################
+
+MODEL = 'vehicle.lincoln.mkz_2017' #'vehicle.toyota.prius'
+MODEL_ADV = 'vehicle.lincoln.mkz_2017'
+
+EGO_INIT_DIST = [30, 40]
+param EGO_SPEED = VerifaiRange(7, 10)
+param EGO_BRAKE = VerifaiRange(0.8, 1.0)
+
+param ADV1_DIST = VerifaiRange(6, 10)
+ADV_INIT_DIST = [15, 25]
+param ADV_SPEED = VerifaiRange(5, 8)
+
+PED_MIN_SPEED = 1.0
+PED_THRESHOLD = 20
+PED_FINAL_SPEED = 1.0
+
+SAFETY_DIST = 8
+CRASH_DIST = 5
+TERM_DIST = 80
+
+#################################
+# AGENT BEHAVIORS #
+#################################
+
+behavior EgoBehavior(trajectory):
+ try:
+ do FollowTrajectoryBehavior(target_speed=globalParameters.EGO_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.EGO_SPEED)
+ interrupt when withinDistanceToAnyObjs(self, SAFETY_DIST):
+ take SetBrakeAction(globalParameters.EGO_BRAKE)
+
+behavior Adv1Behavior(trajectory):
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+
+behavior Adv2Behavior(trajectory):
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+
+behavior Adv3Behavior(trajectory):
+ do FollowTrajectoryBehavior(target_speed=globalParameters.ADV_SPEED, trajectory=trajectory)
+ do FollowLaneBehavior(target_speed=globalParameters.ADV_SPEED)
+
+#################################
+# SPATIAL RELATIONS #
+#################################
+
+intersection = Uniform(*filter(lambda i: i.is4Way, network.intersections))
+
+# ego: straight from S to N
+egoManeuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, intersection.maneuvers))
+egoInitLane = egoManeuver.startLane
+egoTrajectory = [egoInitLane, egoManeuver.connectingLane, egoManeuver.endLane]
+egoSpawnPt = new OrientedPoint in egoInitLane.centerline
+
+# adv1: straight from S to N
+adv1InitLane = egoInitLane
+adv1Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv1InitLane.maneuvers))
+adv1Trajectory = [adv1InitLane, adv1Maneuver.connectingLane, adv1Maneuver.endLane]
+
+# adv2: straight from W to E
+adv2InitLane = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT,
+ Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, egoInitLane.maneuvers)).conflictingManeuvers)).startLane
+adv2Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv2InitLane.maneuvers))
+adv2Trajectory = [adv2InitLane, adv2Maneuver.connectingLane, adv2Maneuver.endLane]
+adv2SpawnPt = new OrientedPoint in adv2InitLane.centerline
+
+# adv3: straight from E to W
+adv3InitLane = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv2Maneuver.reverseManeuvers)).startLane
+adv3Maneuver = Uniform(*filter(lambda m: m.type is ManeuverType.STRAIGHT, adv3InitLane.maneuvers))
+adv3Trajectory = [adv3InitLane, adv3Maneuver.connectingLane, adv3Maneuver.endLane]
+adv3SpawnPt = new OrientedPoint in adv3InitLane.centerline
+
+#################################
+# SCENARIO SPECIFICATION #
+#################################
+
+ego = new Car at egoSpawnPt,
+ with blueprint MODEL,
+ with behavior EgoBehavior(egoTrajectory)
+
+adv1 = new Car following roadDirection for globalParameters.ADV1_DIST,
+ with blueprint MODEL_ADV,
+ with behavior Adv1Behavior(adv1Trajectory)
+
+adv2 = new Car at adv2SpawnPt,
+ with blueprint MODEL_ADV,
+ with behavior Adv2Behavior(adv2Trajectory)
+
+adv3 = new Car at adv3SpawnPt,
+ with blueprint MODEL_ADV,
+ with behavior Adv3Behavior(adv3Trajectory)
+
+require EGO_INIT_DIST[0] <= (distance to intersection) <= EGO_INIT_DIST[1]
+require ADV_INIT_DIST[0] <= (distance from adv2 to intersection) <= ADV_INIT_DIST[1]
+require ADV_INIT_DIST[0] <= (distance from adv3 to intersection) <= ADV_INIT_DIST[1]
+terminate when (distance to egoSpawnPt) > TERM_DIST
+
+#################################
+# RECORDING #
+#################################
+
+record (ego in network.drivableRegion) as egoIsInDrivableRegion
+record (distance from ego to network.drivableRegion) as egoDistToDrivableRegion
+record (distance from ego to egoInitLane.group) as egoDistToEgoInitLane
+record (distance from ego to egoManeuver.endLane.group) as egoDistToEgoEndLane
+record (distance from ego to ego.lane.centerline) as egoDistToEgoLaneCenterline
+record (distance from ego to intersection) as egoDistToIntersection
+
+record (distance from ego to adv1) as egoDistToAdv1
+record (distance to egoSpawnPt) as egoDistToEgoSpawnPt
+
+record ego._boundingPolygon as egoPoly
+record adv1._boundingPolygon as adv1Poly
+record adv2._boundingPolygon as adv2Poly
+record adv3._boundingPolygon as adv3Poly
+record ego.lane.polygon as egoLanePoly
+record adv1.lane.polygon as adv1LanePoly
+record adv2.lane.polygon as adv2LanePoly
+record adv3.lane.polygon as adv3LanePoly
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight.sgraph b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight.sgraph
new file mode 100644
index 0000000..eb19a9a
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight.sgraph
@@ -0,0 +1,23 @@
+# ID 0
+# Node list
+0 off rule0 monitor
+1 on rule1 monitor
+2 off rule2 monitor
+3 off rule3 monitor
+4 off rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 on rule8 monitor
+# Edge list
+0 3
+1 3
+2 3
+3 4
+3 5
+4 7
+4 8
+5 7
+5 8
+7 6
+8 6
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_00.graph b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_00.graph
new file mode 100644
index 0000000..a43073c
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_00.graph
@@ -0,0 +1,16 @@
+# ID 0
+# Node list
+0 off rule0 monitor
+1 on rule1 monitor
+2 off rule2 monitor
+3 off rule3 monitor
+4 off rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 on rule8 monitor
+# Edge list
+0 3
+3 4
+4 7
+7 6
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_01.graph b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_01.graph
new file mode 100644
index 0000000..e05f098
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_01.graph
@@ -0,0 +1,16 @@
+# ID 1
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+2 on rule2 monitor
+3 on rule3 monitor
+4 on rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 off rule8 monitor
+# Edge list
+0 3
+1 3
+2 3
+3 8
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_02.graph b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_02.graph
new file mode 100644
index 0000000..c762bbe
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_02.graph
@@ -0,0 +1,15 @@
+# ID 2
+# Node list
+0 on rule0 monitor
+1 on rule1 monitor
+2 on rule2 monitor
+3 on rule3 monitor
+4 on rule4 monitor
+5 on rule5 monitor
+6 off rule6 monitor
+7 off rule7 monitor
+8 off rule8 monitor
+# Edge list
+0 3
+3 5
+5 6
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_rulebook.py b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_rulebook.py
new file mode 100644
index 0000000..ac54f1c
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_rulebook.py
@@ -0,0 +1,58 @@
+import numpy as np
+
+from verifai.rulebook import rulebook
+
+class rulebook_multistraight(rulebook):
+ iteration = 0
+
+ def __init__(self, graph_path, rule_file, save_path=None, single_graph=False, using_sampler=-1, exploration_ratio=2.0):
+ rulebook.using_sampler = using_sampler
+ rulebook.exploration_ratio = exploration_ratio
+ super().__init__(graph_path, rule_file, single_graph=single_graph)
+ self.save_path = save_path
+
+ def evaluate(self, simulation):
+ # Extract trajectory information
+ positions = np.array(simulation.result.trajectory)
+ ego_dist_to_intersection = np.array(simulation.result.records["egoDistToIntersection"])
+
+ # Find switching points, i.e., ego has reached the intersection / ego has passed the intersection
+ switch_idx_1 = len(simulation.result.trajectory)
+ switch_idx_2 = len(simulation.result.trajectory)
+ for i in range(len(ego_dist_to_intersection)):
+ if ego_dist_to_intersection[i][1] == 0 and switch_idx_1 == len(simulation.result.trajectory):
+ switch_idx_1 = i
+ break
+ if switch_idx_1 < len(simulation.result.trajectory):
+ for i in reversed(range(switch_idx_1, len(ego_dist_to_intersection))):
+ if ego_dist_to_intersection[i][1] == 0:
+ switch_idx_2 = i + 1
+ break
+ assert switch_idx_1 <= switch_idx_2
+
+ # Evaluation
+ indices_0 = np.arange(0, switch_idx_1)
+ indices_1 = np.arange(switch_idx_1, switch_idx_2)
+ indices_2 = np.arange(switch_idx_2, len(simulation.result.trajectory))
+ #print('Indices:', indices_0, indices_1, indices_2)
+ if self.single_graph:
+ rho0 = self.evaluate_segment(simulation, 0, indices_0)
+ rho1 = self.evaluate_segment(simulation, 0, indices_1)
+ rho2 = self.evaluate_segment(simulation, 0, indices_2)
+ print('Actual rho:')
+ for r in rho0:
+ print(r, end=' ')
+ print()
+ for r in rho1:
+ print(r, end=' ')
+ print()
+ for r in rho2:
+ print(r, end=' ')
+ print()
+ rho = self.evaluate_segment(simulation, 0, np.arange(0, len(simulation.result.trajectory)))
+ return np.array([rho])
+ rho0 = self.evaluate_segment(simulation, 0, indices_0)
+ rho1 = self.evaluate_segment(simulation, 1, indices_1)
+ rho2 = self.evaluate_segment(simulation, 2, indices_2)
+ return np.array([rho0, rho1, rho2])
+
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_spec.py b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_spec.py
new file mode 100644
index 0000000..25680d5
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2straight/multi_verifai2straight_spec.py
@@ -0,0 +1,74 @@
+import numpy as np
+
+def rule0(simulation, indices): # B, 1: safe distance to adv1
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [1], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule1(simulation, indices): # B, 2: safe distance to adv2
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [2], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule2(simulation, indices): # B, 3: safe distance to adv3
+ if indices.size == 0:
+ return 1
+ positions = np.array(simulation.result.trajectory)
+ distances_to_adv = positions[indices, [0], :] - positions[indices, [3], :]
+ distances_to_adv = np.linalg.norm(distances_to_adv, axis=1)
+ rho = np.min(distances_to_adv, axis=0) - 8
+ return rho
+
+def rule3(simulation, indices): # C: stay in drivable area
+ if indices.size == 0:
+ return 1
+ distance_to_drivable = np.array(simulation.result.records["egoDistToDrivableRegion"])
+ rho = -np.max(distance_to_drivable[indices], axis=0)[1]
+ return rho
+
+def rule4(simulation, indices): # D, 1: stay in the correct side of the road, before intersection
+ if indices.size == 0:
+ return 1
+ distance_to_lane_group = np.array(simulation.result.records["egoDistToEgoInitLane"])
+ rho = -np.max(distance_to_lane_group[indices], axis=0)[1]
+ return rho
+
+def rule5(simulation, indices): # D, 2: stay in the correct side of the road, after intersection
+ if indices.size == 0:
+ return 1
+ distance_to_lane_group = np.array(simulation.result.records["egoDistToEgoEndLane"])
+ rho = -np.max(distance_to_lane_group[indices], axis=0)[1]
+ return rho
+
+def rule6(simulation, indices): # F: lane keeping
+ if indices.size == 0:
+ return 1
+ distance_to_lane_center = np.array(simulation.result.records["egoDistToEgoLaneCenterline"])
+ rho = 0.4 - np.max(distance_to_lane_center[indices], axis=0)[1]
+ return rho
+
+def rule7(simulation, indices): # H, 1: reach intersection
+ if indices.size == 0:
+ return 1
+ if max(indices) < len(simulation.result.trajectory) - 1:
+ return 1
+ ego_dist_to_intersection = np.array(simulation.result.records["egoDistToIntersection"])
+ rho = -np.min(ego_dist_to_intersection[indices], axis=0)[1]
+ return rho
+
+def rule8(simulation, indices): # H, 2: reach end lane
+ if indices.size == 0:
+ return 1
+ if max(indices) < len(simulation.result.trajectory) - 1:
+ return 1
+ ego_dist_to_end_lane = np.array(simulation.result.records["egoDistToEgoEndLane"])
+ rho = -np.min(ego_dist_to_end_lane[indices], axis=0)[1]
+ return rho
\ No newline at end of file
diff --git a/examples/dynamic_rulebook/multi_verifai2straight/util/multi_verifai2straight_analyze_diversity.py b/examples/dynamic_rulebook/multi_verifai2straight/util/multi_verifai2straight_analyze_diversity.py
new file mode 100644
index 0000000..a721de5
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2straight/util/multi_verifai2straight_analyze_diversity.py
@@ -0,0 +1,48 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+directory = sys.argv[1]
+all_files = os.listdir(directory)
+all_files = [f for f in all_files if f.endswith('.csv') and f.startswith(sys.argv[2]+'.')]
+mode = sys.argv[3] # multi / single
+
+fig = plt.figure()
+ax = fig.add_subplot(projection='3d')
+count = 0
+ego_speed = []
+ego_brake = []
+adv_speed = []
+adv1_dist = []
+for file in all_files:
+ infile = open(directory+'/'+file, 'r')
+ lines = infile.readlines()
+ if mode == 'single':
+ for i in range(1, len(lines)):
+ line = lines[i] #TODO: identify the counterexamples
+ ego_speed.append(float(line.split(',')[-10]))
+ ego_brake.append(float(line.split(',')[-11]))
+ adv_speed.append(float(line.split(',')[-12]))
+ adv1_dist.append(float(line.split(',')[-13]))
+ else:
+ for i in range(1, len(lines), 3):
+ line1 = lines[i]
+ line2 = lines[i+1]
+ line3 = lines[i+2] #TODO: identify the counterexamples
+ ego_speed.append(float(line1.split(',')[-10]))
+ ego_brake.append(float(line1.split(',')[-11]))
+ adv_speed.append(float(line1.split(',')[-12]))
+ adv1_dist.append(float(line1.split(',')[-13]))
+
+ax.scatter(ego_speed, adv_speed, adv1_dist)
+ax.set_xlabel('EGO_SPEED')
+ax.set_ylabel('ADV_SPEED')
+ax.set_zlabel('ADV1_DIST')
+plt.savefig(directory+'/'+sys.argv[2]+'_scatter.png')
+
+print("Standard deviation of ego_speed:", np.std(ego_speed), len(ego_speed))
+print("Standard deviation of adv_speed:", np.std(adv_speed), len(adv_speed))
+print("Standard deviation of ego_brake:", np.std(ego_brake), len(ego_brake))
+print("Standard deviation of adv1_dist:", np.std(adv1_dist), len(adv1_dist))
+print()
diff --git a/examples/dynamic_rulebook/multi_verifai2straight/util/multi_verifai2straight_collect_result.py b/examples/dynamic_rulebook/multi_verifai2straight/util/multi_verifai2straight_collect_result.py
new file mode 100644
index 0000000..3fc0a47
--- /dev/null
+++ b/examples/dynamic_rulebook/multi_verifai2straight/util/multi_verifai2straight_collect_result.py
@@ -0,0 +1,144 @@
+import sys
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import itertools
+
+infile = open(sys.argv[1], 'r') # *.txt
+mode = sys.argv[2] # multi / single
+order = sys.argv[3] # alternate / sequential
+
+# error weights
+result_count_0 = [[] for i in range(3)]
+result_count_1 = [[] for i in range(3)]
+result_count_2 = [[] for i in range(3)]
+# counterexample types
+counterexample_type_0 = [{} for i in range(3)]
+counterexample_type_1 = [{} for i in range(3)]
+counterexample_type_2 = [{} for i in range(3)]
+curr_source = 0
+lines = infile.readlines()
+infile.close()
+
+for i in range(len(lines)):
+ if mode == 'multi':
+ if 'RHO' in lines[i]:
+ line = lines[i+1].strip().split(' ')
+ val1 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val1.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val1) == 9, 'Invalid length of rho'
+ result_count_0[curr_source].append(val1[0]*16 + val1[3]*8 + val1[4]*4 + val1[7]*2 + val1[6]*1)
+ if tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]])) in counterexample_type_0[curr_source]:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]]))] += 1
+ else:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]]))] = 1
+
+ line = lines[i+2].strip().split(' ')
+ val2 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val2.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val2) == 9, 'Invalid length of rho'
+ result_count_1[curr_source].append(val2[0]*4 + val2[1]*4 + val2[2]*4 + val2[3]*2 + val2[8]*1)
+ if tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]])) in counterexample_type_1[curr_source]:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]]))] += 1
+ else:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]]))] = 1
+
+ line = lines[i+3].strip().split(' ')
+ val3 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val3.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val3) == 9, 'Invalid length of rho'
+ result_count_2[curr_source].append(val3[0]*8 + val3[3]*4 + val3[5]*2 + val3[6]*1)
+ if tuple(1*np.array([val3[0], val3[3], val3[5], val3[6]])) in counterexample_type_2[curr_source]:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[0], val3[3], val3[5], val3[6]]))] += 1
+ else:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[0], val3[3], val3[5], val3[6]]))] = 1
+
+ if order == '-1':
+ curr_source = curr_source + 1 if curr_source < 2 else 0
+ else:
+ if 'Actual rho' in lines[i]:
+ line = lines[i+1].strip().split(' ')
+ val1 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val1.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val1) == 9, 'Invalid length of rho'
+ result_count_0[curr_source].append(val1[0]*16 + val1[3]*8 + val1[4]*4 + val1[7]*2 + val1[6]*1)
+ if tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]])) in counterexample_type_0[curr_source]:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]]))] += 1
+ else:
+ counterexample_type_0[curr_source][tuple(1*np.array([val1[0], val1[3], val1[4], val1[7], val1[6]]))] = 1
+
+ line = lines[i+2].strip().split(' ')
+ val2 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val2.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val2) == 9, 'Invalid length of rho'
+ result_count_1[curr_source].append(val2[0]*4 + val2[1]*4 + val2[2]*4 + val2[3]*2 + val2[8]*1)
+ if tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]])) in counterexample_type_1[curr_source]:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]]))] += 1
+ else:
+ counterexample_type_1[curr_source][tuple(1*np.array([val2[0], val2[1], val2[2], val2[3], val2[8]]))] = 1
+
+ line = lines[i+3].strip().split(' ')
+ val3 = []
+ val_print = []
+ for s in line:
+ if s != '':
+ val3.append(float(s) < 0)
+ val_print.append(float(s))
+ assert len(val3) == 9, 'Invalid length of rho'
+ result_count_2[curr_source].append(val3[0]*8 + val3[3]*4 + val3[5]*2 + val3[6]*1)
+ if tuple(1*np.array([val3[0], val3[3], val3[5], val3[6]])) in counterexample_type_2[curr_source]:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[0], val3[3], val3[5], val3[6]]))] += 1
+ else:
+ counterexample_type_2[curr_source][tuple(1*np.array([val3[0], val3[3], val3[5], val3[6]]))] = 1
+
+ if order == '-1':
+ curr_source = curr_source + 1 if curr_source < 2 else 0
+
+print('Error weights')
+print('segment 0:')
+for i in range(1):
+ print('average:', np.mean(result_count_0[i]), 'max:', np.max(result_count_0[i]), 'percentage:', float(np.count_nonzero(result_count_0[i])/len(result_count_0[i])), result_count_0[i])
+print('segment 1:')
+for i in range(1):
+ print('average:', np.mean(result_count_1[i]), 'max:', np.max(result_count_1[i]), 'percentage:', float(np.count_nonzero(result_count_1[i])/len(result_count_1[i])), result_count_1[i])
+print('segment 2:')
+for i in range(1):
+ print('average:', np.mean(result_count_2[i]), 'max:', np.max(result_count_2[i]), 'percentage:', float(np.count_nonzero(result_count_2[i])/len(result_count_2[i])), result_count_2[i])
+
+print('\nCounterexample types')
+print('segment 0:')
+for i in range(1):
+ print('Types:', len(counterexample_type_0[i]))
+ for key, value in reversed(sorted(counterexample_type_0[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print('segment 1:')
+for i in range(1):
+ print('Types:', len(counterexample_type_1[i]))
+ for key, value in reversed(sorted(counterexample_type_1[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print('segment 2:')
+for i in range(1):
+ print('Types:', len(counterexample_type_2[i]))
+ for key, value in reversed(sorted(counterexample_type_2[i].items(), key=lambda x: x[0])):
+ print("{} : {}".format(key, value))
+print()
diff --git a/examples/dynamic_rulebook/run_multi_01.sh b/examples/dynamic_rulebook/run_multi_01.sh
new file mode 100644
index 0000000..db62a32
--- /dev/null
+++ b/examples/dynamic_rulebook/run_multi_01.sh
@@ -0,0 +1,35 @@
+iteration=3
+scenario='multi_01'
+log_file="result_${scenario}_demab0.log"
+result_file="result_${scenario}_demab0.txt"
+csv_file="result_${scenario}_demab0"
+sampler_idx=0 # 0 / 1 / -1 (-1 is for alternate)
+sampler_type=demab # demab / dmab / random / dce / halton / udemab
+exploration_ratio=2.0
+simulator=scenic.simulators.metadrive.model
+use_dynamic_rulebook=true # true / false (false is for a monolithic rulebook)
+
+rm $scenario/outputs/$log_file
+rm $scenario/outputs/$result_file
+rm $scenario/outputs/$csv_file.*csv
+rm $scenario/outputs/$csv_file\_scatter.png
+if [ "$use_dynamic_rulebook" = true ]; then
+
+ for seed in $(seq 0 2);
+ do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic -gp $scenario/ -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator -co $scenario/outputs --exploration-ratio $exploration_ratio >> $scenario/outputs/$log_file
+ done
+
+ python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file multi $sampler_idx >> $scenario/outputs/$result_file
+ python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file multi >> $scenario/outputs/$result_file
+
+else
+
+ for seed in $(seq 0 2);
+ do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic --single-graph -gp $scenario/$scenario.sgraph -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator -co $scenario/outputs --exploration-ratio $exploration_ratio >> $scenario/outputs/$log_file
+ done
+
+ python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file single $sampler_idx >> $scenario/outputs/$result_file
+ python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file single >> $scenario/outputs/$result_file
+fi
diff --git a/examples/dynamic_rulebook/run_multi_02.sh b/examples/dynamic_rulebook/run_multi_02.sh
new file mode 100644
index 0000000..74b9c40
--- /dev/null
+++ b/examples/dynamic_rulebook/run_multi_02.sh
@@ -0,0 +1,36 @@
+iteration=3
+scenario='multi_02'
+log_file="result_${scenario}_demab0.log"
+result_file="result_${scenario}_demab0.txt"
+csv_file="result_${scenario}_demab0"
+sampler_idx=0 # 0 / 1 / -1 (-1 is for alternate)
+sampler_type=demab # demab / dmab / random / dce / halton / udemab
+exploration_ratio=2.0
+simulator=scenic.simulators.metadrive.model
+use_dynamic_rulebook=true # true / false (false is for a monolithic rulebook)
+
+rm $scenario/outputs/$log_file
+rm $scenario/outputs/$result_file
+rm $scenario/outputs/$csv_file.*csv
+rm $scenario/outputs/$csv_file\_scatter.png
+if [ "$use_dynamic_rulebook" = true ]; then
+
+ for seed in $(seq 0 2);
+ do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic -gp $scenario/ -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator -co $scenario/outputs --exploration-ratio $exploration_ratio >> $scenario/outputs/$log_file
+ #python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic -gp $scenario/ -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator -co $scenario/outputs --exploration-ratio $exploration_ratio --using-continuous --use-dependency >> $scenario/outputs/$log_file
+ done
+
+ python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file multi $sampler_idx >> $scenario/outputs/$result_file
+ python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file multi >> $scenario/outputs/$result_file
+
+else
+
+ for seed in $(seq 0 2);
+ do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic --single-graph -gp $scenario/$scenario.sgraph -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator -co $scenario/outputs --exploration-ratio $exploration_ratio >> $scenario/outputs/$log_file
+ done
+
+ python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file single $sampler_idx >> $scenario/outputs/$result_file
+ python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file single >> $scenario/outputs/$result_file
+fi
diff --git a/examples/dynamic_rulebook/run_multi_03.sh b/examples/dynamic_rulebook/run_multi_03.sh
new file mode 100644
index 0000000..b10bab6
--- /dev/null
+++ b/examples/dynamic_rulebook/run_multi_03.sh
@@ -0,0 +1,36 @@
+iteration=3
+scenario='multi_03'
+log_file="result_${scenario}_demab0.log"
+result_file="result_${scenario}_demab0.txt"
+csv_file="result_${scenario}_demab0"
+sampler_idx=0 # 0 / 1 / 2 / -1 (-1 is for alternate)
+sampler_type=demab # demab / dmab / random / dce / halton / udemab
+exploration_ratio=2.0
+simulator=scenic.simulators.metadrive.model
+use_dynamic_rulebook=true # true / false (false is for a monolithic rulebook)
+simulation_steps=300
+
+rm $scenario/outputs/$log_file
+rm $scenario/outputs/$result_file
+rm $scenario/outputs/$csv_file.*csv
+rm $scenario/outputs/$csv_file\_scatter.png
+if [ "$use_dynamic_rulebook" = true ]; then
+
+ for seed in $(seq 0 2);
+ do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic -gp $scenario/ -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator --max-simulation-steps $simulation_steps -co $scenario/outputs --exploration-ratio $exploration_ratio >> $scenario/outputs/$log_file
+ done
+
+ python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file multi $sampler_idx >> $scenario/outputs/$result_file
+ python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file multi >> $scenario/outputs/$result_file
+
+else
+
+ for seed in $(seq 0 2);
+ do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic --single-graph -gp $scenario/$scenario.sgraph -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator --max-simulation-steps $simulation_steps -co $scenario/outputs --exploration-ratio $exploration_ratio >> $scenario/outputs/$log_file
+ done
+
+ python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file single $sampler_idx >> $scenario/outputs/$result_file
+ python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file single >> $scenario/outputs/$result_file
+fi
diff --git a/examples/dynamic_rulebook/run_multi_04.sh b/examples/dynamic_rulebook/run_multi_04.sh
new file mode 100644
index 0000000..8be3149
--- /dev/null
+++ b/examples/dynamic_rulebook/run_multi_04.sh
@@ -0,0 +1,20 @@
+iteration=3
+scenario='multi_04'
+log_file="result_${scenario}_demab.log"
+result_file="result_${scenario}_demab.txt"
+csv_file="result_${scenario}_demab"
+sampler_idx=0 # 0 / 1 / -1 (-1 is for alternate)
+sampler_type=demab # demab / dmab / random / dce / halton / udemab
+simulator=scenic.simulators.metadrive.model
+simulation_steps=200
+
+rm $scenario/outputs/$log_file
+rm $scenario/outputs/$result_file
+rm $scenario/outputs/$csv_file.*csv
+rm $scenario/outputs/$csv_file\_scatter.png
+for seed in $(seq 0 2);
+do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic -gp $scenario/ -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator --max-simulation-steps $simulation_steps -co $scenario/outputs >> $scenario/outputs/$log_file
+done
+python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file multi $sampler_idx >> $scenario/outputs/$result_file
+python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file multi >> $scenario/outputs/$result_file
diff --git a/examples/dynamic_rulebook/run_multi_verifai2left.sh b/examples/dynamic_rulebook/run_multi_verifai2left.sh
new file mode 100644
index 0000000..19598e2
--- /dev/null
+++ b/examples/dynamic_rulebook/run_multi_verifai2left.sh
@@ -0,0 +1,36 @@
+iteration=3
+scenario='multi_verifai2left'
+log_file="result_${scenario}_demab0.log"
+result_file="result_${scenario}_demab0.txt"
+csv_file="result_${scenario}_demab0"
+sampler_idx=0 # 0 / 1 / 2 / -1 (-1 is for alternate)
+sampler_type=demab # demab / dmab / random / dce / halton / udemab
+exploration_ratio=2.0
+simulator=scenic.simulators.metadrive.model
+use_dynamic_rulebook=true # true / false (false is for a monolithic rulebook)
+simulation_steps=200
+
+rm $scenario/outputs/$log_file
+rm $scenario/outputs/$result_file
+rm $scenario/outputs/$csv_file.*csv
+rm $scenario/outputs/$csv_file\_scatter.png
+if [ "$use_dynamic_rulebook" = true ]; then
+
+ for seed in $(seq 0 2);
+ do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic -gp $scenario/ -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator --max-simulation-steps $simulation_steps -co $scenario/outputs --exploration-ratio $exploration_ratio >> $scenario/outputs/$log_file
+ done
+
+ python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file multi $sampler_idx >> $scenario/outputs/$result_file
+ python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file multi >> $scenario/outputs/$result_file
+
+else
+
+ for seed in $(seq 0 2);
+ do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic --single-graph -gp $scenario/$scenario.sgraph -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator --max-simulation-steps $simulation_steps -co $scenario/outputs --exploration-ratio $exploration_ratio >> $scenario/outputs/$log_file
+ done
+
+ python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file single $sampler_idx >> $scenario/outputs/$result_file
+ python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file single >> $scenario/outputs/$result_file
+fi
diff --git a/examples/dynamic_rulebook/run_multi_verifai2right.sh b/examples/dynamic_rulebook/run_multi_verifai2right.sh
new file mode 100644
index 0000000..b2c14ea
--- /dev/null
+++ b/examples/dynamic_rulebook/run_multi_verifai2right.sh
@@ -0,0 +1,36 @@
+iteration=3
+scenario='multi_verifai2right'
+log_file="result_${scenario}_demab0.log"
+result_file="result_${scenario}_demab0.txt"
+csv_file="result_${scenario}_demab0"
+sampler_idx=0 # 0 / 1 / 2 / -1 (-1 is for alternate)
+sampler_type=demab # demab / dmab / random / dce / halton / udemab
+exploration_ratio=2.0
+simulator=scenic.simulators.metadrive.model
+use_dynamic_rulebook=true # true / false (false is for a monolithic rulebook)
+simulation_steps=200
+
+rm $scenario/outputs/$log_file
+rm $scenario/outputs/$result_file
+rm $scenario/outputs/$csv_file.*csv
+rm $scenario/outputs/$csv_file\_scatter.png
+if [ "$use_dynamic_rulebook" = true ]; then
+
+ for seed in $(seq 0 2);
+ do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic -gp $scenario/ -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator --max-simulation-steps $simulation_steps -co $scenario/outputs --exploration-ratio $exploration_ratio >> $scenario/outputs/$log_file
+ done
+
+ python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file multi $sampler_idx >> $scenario/outputs/$result_file
+ python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file multi >> $scenario/outputs/$result_file
+
+else
+
+ for seed in $(seq 0 2);
+ do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic --single-graph -gp $scenario/$scenario.sgraph -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator --max-simulation-steps $simulation_steps -co $scenario/outputs --exploration-ratio $exploration_ratio >> $scenario/outputs/$log_file
+ done
+
+ python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file single $sampler_idx >> $scenario/outputs/$result_file
+ python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file single >> $scenario/outputs/$result_file
+fi
diff --git a/examples/dynamic_rulebook/run_multi_verifai2straight.sh b/examples/dynamic_rulebook/run_multi_verifai2straight.sh
new file mode 100644
index 0000000..e35ae61
--- /dev/null
+++ b/examples/dynamic_rulebook/run_multi_verifai2straight.sh
@@ -0,0 +1,36 @@
+iteration=3
+scenario='multi_verifai2straight'
+log_file="result_${scenario}_demab0.log"
+result_file="result_${scenario}_demab0.txt"
+csv_file="result_${scenario}_demab0"
+sampler_idx=0 # 0 / 1 / 2 / -1 (-1 is for alternate)
+sampler_type=demab # demab / dmab / random / dce / halton / udemab
+exploration_ratio=2.0
+simulator=scenic.simulators.metadrive.model
+use_dynamic_rulebook=true # true / false (false is for a monolithic rulebook)
+simulation_steps=200
+
+rm $scenario/outputs/$log_file
+rm $scenario/outputs/$result_file
+rm $scenario/outputs/$csv_file.*csv
+rm $scenario/outputs/$csv_file\_scatter.png
+if [ "$use_dynamic_rulebook" = true ]; then
+
+ for seed in $(seq 0 2);
+ do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic -gp $scenario/ -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator --max-simulation-steps $simulation_steps -co $scenario/outputs --exploration-ratio $exploration_ratio >> $scenario/outputs/$log_file
+ done
+
+ python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file multi $sampler_idx >> $scenario/outputs/$result_file
+ python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file multi >> $scenario/outputs/$result_file
+
+else
+
+ for seed in $(seq 0 2);
+ do
+ python $scenario/$scenario.py -n $iteration --headless -e $csv_file.$seed -sp $scenario/$scenario.scenic --single-graph -gp $scenario/$scenario.sgraph -rp $scenario/$scenario\_spec.py -s $sampler_type --seed $seed --using-sampler $sampler_idx -m $simulator --max-simulation-steps $simulation_steps -co $scenario/outputs --exploration-ratio $exploration_ratio >> $scenario/outputs/$log_file
+ done
+
+ python $scenario/util/$scenario\_collect_result.py $scenario/outputs/$log_file single $sampler_idx >> $scenario/outputs/$result_file
+ python $scenario/util/$scenario\_analyze_diversity.py $scenario/outputs/ $csv_file single >> $scenario/outputs/$result_file
+fi
diff --git a/src/verifai/error_table.py b/src/verifai/error_table.py
index 600087f..861302a 100644
--- a/src/verifai/error_table.py
+++ b/src/verifai/error_table.py
@@ -38,7 +38,7 @@ def update_column_names(self, column_names):
self.table.columns = column_names
self.column_names = column_names
- def update_error_table(self, sample, rho):
+ def update_error_table(self, sample, rho, is_multi=False):
sample = self.space.flatten(sample, fixedDimension=True)
sample_dict = {}
for k, v in zip(self.table.columns, list(sample)):
@@ -46,7 +46,7 @@ def update_error_table(self, sample, rho):
locs = np.where(np.array(sample) == None)
self.ignore_locs = self.ignore_locs + list(locs[0])
sample_dict[k] = float(v) if self.column_type[k] and v is not None else v
- if isinstance(rho, (list, tuple)):
+ if is_multi or isinstance(rho, (list, tuple)):
for i,r in enumerate(rho[:-1]):
if "rho_" + str(i) not in self.column_names:
self.column_names.append("rho_"+str(i))
diff --git a/src/verifai/falsifier.py b/src/verifai/falsifier.py
index fe2caef..1887764 100644
--- a/src/verifai/falsifier.py
+++ b/src/verifai/falsifier.py
@@ -4,6 +4,7 @@
from verifai.samplers import TerminationException
from dotmap import DotMap
from verifai.monitor import mtl_specification, specification_monitor, multi_objective_monitor
+from verifai.rulebook import rulebook
from verifai.error_table import error_table
import numpy as np
import progressbar
@@ -36,9 +37,12 @@ def __init__(self, monitor, sampler_type=None, sampler=None, sample_space=None,
params.update(falsifier_params)
if params.sampler_params is None:
params.sampler_params = DotMap(thres=params.fal_thres)
- self.multi = isinstance(self.monitor, multi_objective_monitor)
- if self.multi:
+ self.multi = isinstance(self.monitor, multi_objective_monitor) or isinstance(self.monitor, rulebook)
+ self.dynamic = isinstance(self.monitor, rulebook)
+ if isinstance(self.monitor, multi_objective_monitor):
params.sampler_params.priority_graph = self.monitor.graph
+ elif isinstance(self.monitor, rulebook):
+ pass
self.save_error_table = params.save_error_table
self.save_safe_table = params.save_safe_table
self.error_table_path = params.error_table_path
@@ -51,7 +55,7 @@ def __init__(self, monitor, sampler_type=None, sampler=None, sample_space=None,
self.sampler_params = params.sampler_params
self.verbosity = params.verbosity
- server_params = DotMap(init=True)
+ server_params = DotMap(init=True, dynamic=self.dynamic)
if server_options is not None:
server_params.update(server_options)
if server_params.init:
@@ -82,11 +86,11 @@ def init_error_table(self):
def populate_error_table(self, sample, rho, error=True):
if error:
- self.error_table.update_error_table(sample, rho)
+ self.error_table.update_error_table(sample, rho, is_multi=self.multi)
if self.error_table_path:
self.write_table(self.error_table.table, self.error_table_path)
else:
- self.safe_table.update_error_table(sample, rho)
+ self.safe_table.update_error_table(sample, rho, is_multi=self.multi)
if self.safe_table_path:
self.write_table(self.safe_table.table, self.safe_table_path)
@@ -157,8 +161,14 @@ def run_falsifier(self):
if self.verbosity >= 1:
print("Sampler has generated all possible samples")
break
- if self.verbosity >= 2:
- print("Sample no: ", i, "\nSample: ", sample, "\nRho: ", rho)
+ if self.verbosity >= 1:
+ print("Sample no: ", i, "\nSample: ", sample, "\nRho: ", rho, "\n")
+ if self.dynamic:
+ print('RHO')
+ for rh in rho:
+ for r in rh:
+ print(r, end=' ')
+ print()
self.samples[i] = sample
server_samples.append(sample)
rhos.append(rho)
@@ -176,15 +186,23 @@ def run_falsifier(self):
bar.finish()
self.server.terminate()
for sample, rho in zip(server_samples, rhos):
- ce = any([r <= self.fal_thres for r in rho]) if self.multi else rho <= self.fal_thres
- if ce:
- if self.save_error_table:
- self.populate_error_table(sample, rho)
- ce_num = ce_num + 1
- if ce_num >= self.ce_num_max:
- break
- elif self.save_safe_table:
- self.populate_error_table(sample, rho, error=False)
+ ce = False
+ if self.dynamic:
+ for r in rho:
+ self.populate_error_table(sample, r)
+ else:
+ if self.multi:
+ ce = any([r <= self.fal_thres for r in rho])
+ else:
+ ce = rho <= self.fal_thres
+ if ce:
+ if self.save_error_table:
+ self.populate_error_table(sample, rho)
+ ce_num = ce_num + 1
+ if ce_num >= self.ce_num_max:
+ break
+ elif self.save_safe_table:
+ self.populate_error_table(sample, rho, error=False)
if self.verbosity >= 1:
print('Falsification complete.')
diff --git a/src/verifai/rulebook.py b/src/verifai/rulebook.py
new file mode 100644
index 0000000..3c689af
--- /dev/null
+++ b/src/verifai/rulebook.py
@@ -0,0 +1,159 @@
+from abc import ABC
+import networkx as nx
+import mtl
+import ast
+import numpy as np
+import os
+
+from verifai.monitor import specification_monitor
+
+class FunctionVisitor(ast.NodeVisitor):
+ def __init__(self):
+ self.functions = []
+
+ def visit_FunctionDef(self, node):
+ self.functions.append(node)
+
+class rulebook(ABC):
+ priority_graphs = {}
+ using_sampler = -1
+ verbosity = 1
+ exploration_ratio = 2.0
+ using_continuous = False
+
+ def __init__(self, graph_path, rule_file, single_graph=False):
+ print('(rulebook.py) Parsing rules...')
+ self._parse_rules(rule_file)
+ print('(rulebook.py) Parsing rulebook...')
+ if single_graph:
+ self._parse_rulebook(graph_path)
+ else:
+ self._parse_rulebooks(graph_path)
+ self.single_graph = single_graph
+
+ def _parse_rules(self, file_path):
+ # Parse the input rules (*_spec.py)
+ with open(file_path, 'r') as file:
+ file_contents = file.read()
+
+ tree = ast.parse(file_contents)
+
+ function_visitor = FunctionVisitor()
+ function_visitor.visit(tree)
+
+ self.functions = {}
+ for function_node in function_visitor.functions:
+ function_name = function_node.name
+ function_code = compile(ast.Module(body=[function_node], type_ignores=[]), '', 'exec')
+ exec(function_code)
+ self.functions[function_name] = locals()[function_name]
+
+ print(f'Parsed functions: {self.functions}')
+
+ def _parse_rulebooks(self, dir):
+ if os.path.isdir(dir):
+ for root, _, files in os.walk(dir):
+ for name in files:
+ fname = os.path.join(root, name)
+ if os.path.splitext(fname)[1] == '.graph':
+ self._parse_rulebook(fname)
+
+ def _parse_rulebook(self, file):
+ # TODO: parse the input rulebook
+ # 1. construct the priority_graph
+ # 2. construct a dictionary mapping from each node_id to corresponding rule object
+ priority_graph = nx.DiGraph()
+ graph_id = -1
+ with open(file, 'r') as f:
+ lines = f.readlines()
+ node_section = False
+ edge_section = False
+ for line in lines:
+ line = line.strip()
+ if line.startswith('# ID'):
+ graph_id = int(line.split(' ')[-1])
+ if self.verbosity >= 1:
+ print(f'Parsing graph {graph_id}')
+ if line == '# Node list':
+ node_section = True
+ continue
+ elif line == '# Edge list':
+ node_section = False
+ edge_section = True
+ continue
+
+ # Node
+ if node_section:
+ node_info = line.split(' ')
+ node_id = int(node_info[0])
+ node_active = True if node_info[1] == 'on' else False
+ rule_name = node_info[2]
+ rule_type = node_info[3]
+ if rule_type == 'monitor':
+ ru = rule(node_id, self.functions[rule_name], rule_type)
+ priority_graph.add_node(node_id, rule=ru, active=node_active, name=rule_name)
+ if self.verbosity >= 2:
+ print(f'Add node {node_id} with rule {rule_name}')
+ #TODO: mtl type
+
+ # Edge
+ if edge_section:
+ edge_info = line.split(' ')
+ src = int(edge_info[0])
+ dst = int(edge_info[1])
+ priority_graph.add_edge(src, dst)
+ if self.verbosity >= 2:
+ print(f'Add edge from {src} to {dst}')
+
+ # TODO: process the graph, e.g., merge the same level nodes
+
+ self.priority_graphs[graph_id] = priority_graph
+
+ def evaluate_segment(self, traj, graph_idx=0, indices=None):
+ # Evaluate the result of each rule on the segment traj[indices] of the trajectory
+ priority_graph = self.priority_graphs[graph_idx]
+ rho = np.ones(len(priority_graph.nodes))
+ idx = 0
+ for id in sorted(priority_graph.nodes):
+ rule = priority_graph.nodes[id]['rule']
+ if priority_graph.nodes[id]['active']:
+ if self.verbosity >= 2:
+ print('Evaluating rule', id)
+ rho[idx] = rule.evaluate(traj, indices)
+ else:
+ rho[idx] = 1
+ idx += 1
+ return rho
+
+ def evaluate_rule(self, traj, rule_id, graph_idx=0, indices=None):
+ # Evaluate the result of a rule on the trajectory
+ priority_graph = self.priority_graphs[graph_idx]
+ rule = priority_graph.nodes[rule_id]['rule']
+ rho = 1
+ if priority_graph.nodes[rule_id]['active']:
+ if self.verbosity >= 2:
+ print('Evaluating rule', rule_id)
+ rho = rule.evaluate(traj, indices)
+ return rho
+
+ def evaluate(self, traj):
+ raise NotImplementedError('evaluate() is not implemented')
+
+ def update_graph(self):
+ pass
+
+class rule(specification_monitor):
+ def __init__(self, node_id, spec, spec_type='monitor'):
+ self.node_id = node_id
+ if spec_type == 'monitor': # spec is a function
+ super().__init__(spec)
+ else: # spec is MTL
+ mtl_specs = [mtl.parse(sp) for sp in spec]
+ mtl_spec = mtl_specs[0]
+ if len(mtl_specs) > 1:
+ for sp in mtl_specs[1:]:
+ mtl_spec = (mtl_spec & sp)
+ super().__init__(mtl_spec)
+
+ def evaluate(self, traj, indices=None):
+ return self.specification(traj, indices)
diff --git a/src/verifai/samplers/domain_sampler.py b/src/verifai/samplers/domain_sampler.py
index 6ead563..eb64e94 100644
--- a/src/verifai/samplers/domain_sampler.py
+++ b/src/verifai/samplers/domain_sampler.py
@@ -165,11 +165,13 @@ def updateVector(self, vector, info, rho):
pass
def set_graph(self, graph):
+ print('(domain_sampler.py) graph =', graph)
self.priority_graph = graph
if graph is not None:
self.thres = [self.thres] * graph.number_of_nodes()
self.num_properties = graph.number_of_nodes()
self.is_multi = True
+ print('(domain_sampler.py) self.num_properties =', self.num_properties)
class DiscreteBoxSampler(DomainSampler):
"""Samplers defined only over discrete hyperboxes"""
diff --git a/src/verifai/samplers/dynamic_ce.py b/src/verifai/samplers/dynamic_ce.py
new file mode 100644
index 0000000..b78427a
--- /dev/null
+++ b/src/verifai/samplers/dynamic_ce.py
@@ -0,0 +1,161 @@
+import numpy as np
+import networkx as nx
+from itertools import product
+from verifai.samplers.domain_sampler import BoxSampler, DiscreteBoxSampler, \
+ DomainSampler, SplitSampler
+from verifai.samplers.random_sampler import RandomSampler
+from verifai.samplers.cross_entropy import DiscreteCrossEntropySampler
+from verifai.samplers.multi_objective import MultiObjectiveSampler
+from verifai.rulebook import rulebook
+
+class DynamicCrossEntropySampler(DomainSampler):
+ def __init__(self, domain, dce_params):
+ print('(dynamic_ce.py) Initializing!!!')
+ print('(dynamic_ce.py) dce_params =', dce_params)
+ super().__init__(domain)
+ self.alpha = dce_params.alpha
+ self.thres = dce_params.thres
+ self.cont_buckets = dce_params.cont.buckets
+ self.cont_dist = dce_params.cont.dist
+ self.disc_dist = dce_params.disc.dist
+ self.cont_ce = lambda domain: ContinuousDynamicCESampler(domain=domain,
+ buckets=self.cont_buckets,
+ dist=self.cont_dist,
+ alpha=self.alpha,
+ thres=self.thres)
+ self.disc_ce = lambda domain: DiscreteDynamicCESampler(domain=domain,
+ dist=self.disc_dist,
+ alpha=self.alpha,
+ thres=self.thres)
+ partition = (
+ (lambda d: d.standardizedDimension > 0, self.cont_ce),
+ (lambda d: d.standardizedIntervals, self.disc_ce)
+ )
+ self.split_samplers = {}
+ for id, priority_graph in rulebook.priority_graphs.items():
+ self.split_samplers[id] = SplitSampler.fromPartition(domain,
+ partition,
+ RandomSampler)
+ for subsampler in self.split_samplers[id].samplers:
+ if isinstance(subsampler, ContinuousDynamicCESampler):
+ print('(dynamic_ce.py) Set priority graph', id)
+ subsampler.set_graph(priority_graph)
+ elif isinstance(subsampler, DiscreteDynamicCESampler):
+ assert True
+ else:
+ assert isinstance(subsampler, RandomSampler)
+ node_ids = list(nx.dfs_preorder_nodes(priority_graph))
+ if not sorted(node_ids) == list(range(len(node_ids))):
+ raise ValueError('Node IDs should be in order and start from 0')
+ if not sorted(list(self.split_samplers.keys())) == list(range(len(rulebook.priority_graphs))):
+ raise ValueError('Priority graph IDs should be in order and start from 0')
+ self.num_segs = len(self.split_samplers)
+ print('(dynamic_ce.py) num_segs =', self.num_segs)
+ self.sampler_idx = 0
+ self.using_sampler = rulebook.using_sampler # -1: round-robin
+ assert self.using_sampler < self.num_segs
+ print('(dynamic_ce.py) using_sampler =', self.using_sampler)
+
+ def getSample(self):
+ if self.using_sampler == -1:
+ # Sample from each segment in a round-robin fashion
+ idx = self.sampler_idx % self.num_segs
+ else:
+ idx = self.using_sampler
+ return self.split_samplers[idx].getSample()
+
+ def update(self, sample, info, rhos):
+ # Update each sampler based on the corresponding segment
+ try:
+ iter(rhos)
+ except:
+ for i in range(len(self.split_samplers)):
+ self.split_samplers[i].update(sample, info, rhos)
+ return
+ if self.using_sampler == -1:
+ print('(dynamic_ce.py) Getting feedback from segment', self.sampler_idx % self.num_segs)
+ for i in range(len(rhos)):
+ self.split_samplers[i].update(sample, info, rhos[i])
+ else:
+ print('(dynamic_ce.py) Getting feedback from segment', self.using_sampler)
+ self.split_samplers[self.using_sampler].update(sample, info, rhos[self.using_sampler])
+ self.sampler_idx += 1
+
+class ContinuousDynamicCESampler(BoxSampler, MultiObjectiveSampler):
+ verbosity = 2
+
+ def __init__(self, domain, alpha, thres,
+ buckets=10, dist=None, restart_every=100):
+ super().__init__(domain)
+ if isinstance(buckets, int):
+ buckets = np.ones(self.dimension) * buckets
+ elif len(buckets) > 1:
+ assert len(buckets) == self.dimension
+ else:
+ buckets = np.ones(self.dimension) * buckets[0]
+ if dist is not None:
+ assert (len(dist) == len(buckets))
+ if dist is None:
+ dist = np.array([np.ones(int(b))/b for b in buckets])
+ self.buckets = buckets # 1*d, each element specifies the number of buckets in that dimension
+ self.dist = dist # N*d, ???
+ self.alpha = alpha
+ self.thres = thres
+ self.current_sample = None
+
+ #self.counts = np.array([np.ones(int(b)) for b in buckets]) # N*d, T (visit times)
+ #self.errors = np.array([np.zeros(int(b)) for b in buckets]) # N*d, total times resulting in maximal counterexample
+ #self.t = 1 # time, used in Q
+ #self.counterexamples = dict()
+ #self.is_multi = True #False
+ #self.invalid = np.array([np.zeros(int(b)) for b in buckets]) # N*d, ???
+ #self.monitor = None
+ #self.rho_values = []
+ #self.restart_every = restart_every
+ #self.exploration_ratio = 2.0
+
+ def getVector(self):
+ return self.generateSample()
+
+ def generateSample(self):
+ bucket_samples = np.array([np.random.choice(int(b), p=self.dist[i])
+ for i, b in enumerate(self.buckets)])
+ self.current_sample = bucket_samples
+ ret = tuple(np.random.uniform(bs, bs+1.)/b for b, bs
+ in zip(self.buckets, bucket_samples))
+ return ret, bucket_samples
+
+ def updateVector(self, vector, info, rho):
+ assert rho is not None
+ self.update_dist_from_multi(vector, info, rho)
+
+ def update_dist_from_multi(self, sample, info, rho):
+ try:
+ iter(rho)
+ except:
+ return
+ if len(rho) != self.num_properties:
+ return
+
+ # AND
+ is_ce = True
+ for node in self.priority_graph.nodes:
+ if self.priority_graph.nodes[node]['active'] and rho[node] >= self.thres[node]:
+ is_ce = False
+ break
+ # OR
+ #is_ce = False
+ #for node in self.priority_graph.nodes:
+ # if self.priority_graph.nodes[node]['active'] and rho[node] < self.thres[node]:
+ # is_ce = True
+ # break
+
+ if not is_ce:
+ return
+ print('(dynamic_ce.py) IS CE! Updating!!!')
+ for row, b in zip(self.dist, info):
+ row *= self.alpha
+ row[b] += 1 - self.alpha
+
+class DiscreteDynamicCESampler(DiscreteCrossEntropySampler):
+ pass
diff --git a/src/verifai/samplers/dynamic_emab.py b/src/verifai/samplers/dynamic_emab.py
new file mode 100644
index 0000000..621bf33
--- /dev/null
+++ b/src/verifai/samplers/dynamic_emab.py
@@ -0,0 +1,253 @@
+import numpy as np
+import networkx as nx
+from itertools import product
+from verifai.samplers.domain_sampler import BoxSampler, DiscreteBoxSampler, \
+ DomainSampler, SplitSampler
+from verifai.samplers.random_sampler import RandomSampler
+from verifai.samplers.cross_entropy import DiscreteCrossEntropySampler
+from verifai.samplers.multi_objective import MultiObjectiveSampler
+from verifai.rulebook import rulebook
+
+class DynamicExtendedMultiArmedBanditSampler(DomainSampler):
+ def __init__(self, domain, demab_params):
+ print('(dynamic_emab.py) Initializing!!!')
+ print('(dynamic_emab.py) demab_params =', demab_params)
+ super().__init__(domain)
+ self.alpha = demab_params.alpha
+ self.thres = demab_params.thres
+ self.cont_buckets = demab_params.cont.buckets
+ self.cont_dist = demab_params.cont.dist
+ self.disc_dist = demab_params.disc.dist
+ self.cont_ce = lambda domain: ContinuousDynamicEMABSampler(domain=domain,
+ buckets=self.cont_buckets,
+ dist=self.cont_dist,
+ alpha=self.alpha,
+ thres=self.thres,
+ exploration_ratio=rulebook.exploration_ratio)
+ self.disc_ce = lambda domain: DiscreteDynamicEMABSampler(domain=domain,
+ dist=self.disc_dist,
+ alpha=self.alpha,
+ thres=self.thres)
+ partition = (
+ (lambda d: d.standardizedDimension > 0, self.cont_ce),
+ (lambda d: d.standardizedIntervals, self.disc_ce)
+ )
+ self.split_samplers = {}
+ for id, priority_graph in rulebook.priority_graphs.items():
+ self.split_samplers[id] = SplitSampler.fromPartition(domain,
+ partition,
+ RandomSampler)
+ for subsampler in self.split_samplers[id].samplers:
+ if isinstance(subsampler, ContinuousDynamicEMABSampler):
+ print('(dynamic_emab.py) Set priority graph', id)
+ subsampler.set_graph(priority_graph)
+ subsampler.compute_error_weight()
+ elif isinstance(subsampler, DiscreteDynamicEMABSampler):
+ assert True
+ else:
+ assert isinstance(subsampler, RandomSampler)
+ node_ids = list(nx.dfs_preorder_nodes(priority_graph))
+ if not sorted(node_ids) == list(range(len(node_ids))):
+ raise ValueError('Node IDs should be in order and start from 0')
+ if not sorted(list(self.split_samplers.keys())) == list(range(len(rulebook.priority_graphs))):
+ raise ValueError('Priority graph IDs should be in order and start from 0')
+ self.num_segs = len(self.split_samplers)
+ print('(dynamic_emab.py) num_segs =', self.num_segs)
+ self.sampler_idx = 0
+ self.using_sampler = rulebook.using_sampler # -1: round-robin
+ assert self.using_sampler < self.num_segs
+ print('(dynamic_emab.py) using_sampler =', self.using_sampler)
+
+ def getSample(self):
+ if self.using_sampler == -1:
+ # Sample from each segment in a round-robin fashion
+ idx = self.sampler_idx % self.num_segs
+ else:
+ idx = self.using_sampler
+ return self.split_samplers[idx].getSample()
+
+ def update(self, sample, info, rhos):
+ # Update each sampler based on the corresponding segment
+ try:
+ iter(rhos)
+ except:
+ for i in range(len(self.split_samplers)):
+ self.split_samplers[i].update(sample, info, rhos)
+ return
+ if self.using_sampler == -1:
+ print('(dynamic_emab.py) Getting feedback from segment', self.sampler_idx % self.num_segs)
+ for i in range(len(rhos)):
+ self.split_samplers[i].update(sample, info, rhos[i])
+ else:
+ print('(dynamic_emab.py) Getting feedback from segment', self.using_sampler)
+ self.split_samplers[self.using_sampler].update(sample, info, rhos[self.using_sampler])
+ self.sampler_idx += 1
+
+class ContinuousDynamicEMABSampler(BoxSampler, MultiObjectiveSampler):
+ verbosity = 1
+
+ def __init__(self, domain, alpha, thres,
+ buckets=10, dist=None, restart_every=100, exploration_ratio=2.0):
+ super().__init__(domain)
+ if isinstance(buckets, int):
+ buckets = np.ones(self.dimension) * buckets
+ elif len(buckets) > 1:
+ assert len(buckets) == self.dimension
+ else:
+ buckets = np.ones(self.dimension) * buckets[0]
+ if dist is not None:
+ assert (len(dist) == len(buckets))
+ if dist is None:
+ dist = np.array([np.ones(int(b))/b for b in buckets])
+ self.buckets = buckets # 1*d, each element specifies the number of buckets in that dimension
+ self.dist = dist # N*d, ???
+ self.alpha = alpha
+ self.thres = thres
+ self.current_sample = None
+ self.counts = np.array([np.ones(int(b)) for b in buckets]) # N*d, T (visit times)
+ self.errors = np.array([np.zeros(int(b)) for b in buckets]) # N*d, total times resulting in maximal counterexample
+ self.t = 1 # time, used in Q
+ self.counterexamples = dict()
+ self.is_multi = True #False
+ self.invalid = np.array([np.zeros(int(b)) for b in buckets]) # N*d, ???
+ self.monitor = None
+ self.rho_values = []
+ self.restart_every = restart_every
+ self.exploration_ratio = exploration_ratio
+
+ def getVector(self):
+ return self.generateSample()
+
+ def generateSample(self):
+ proportions = self.errors / self.counts
+ Q = proportions + np.sqrt(self.exploration_ratio / self.counts * np.log(self.t))
+ # choose the bucket with the highest "goodness" value, breaking ties randomly.
+ bucket_samples = np.array([np.random.choice(np.flatnonzero(np.isclose(Q[i], Q[i].max())))
+ for i in range(len(self.buckets))])
+ self.current_sample = bucket_samples
+ ret = tuple(np.random.uniform(bs, bs+1.)/b for b, bs
+ in zip(self.buckets, bucket_samples)) # uniform randomly sample from the range of the bucket
+ return ret, bucket_samples
+
+ def updateVector(self, vector, info, rho):
+ assert rho is not None
+ # "random restarts" to generate a new topological sort of the priority graph
+ # every restart_every samples.
+ if self.is_multi:
+ if self.monitor is not None and self.monitor.linearize and self.t % self.restart_every == 0:
+ self.monitor._linearize()
+ self.update_dist_from_multi(vector, info, rho)
+ return
+ self.t += 1
+ for i, b in enumerate(info):
+ self.counts[i][b] += 1.
+ if rho < self.thres:
+ self.errors[i][b] += 1.
+
+ def is_better_counterexample(self, ce1, ce2):
+ if ce2 is None:
+ return True
+ return self._compute_error_value(ce1) > self._compute_error_value(ce2)
+
+ def _get_total_counterexamples(self):
+ return sum(self.counterexamples.values())
+
+ def _update_counterexample(self, ce, to_delete=False): # update counterexamples, may or may not delete non-maximal counterexamples
+ if ce in self.counterexamples:
+ return True
+ if to_delete:
+ to_remove = set()
+ if len(self.counterexamples) > 0:
+ for other_ce in self.counterexamples:
+ if self.is_better_counterexample(other_ce, ce):
+ return False
+ for other_ce in self.counterexamples:
+ if self.is_better_counterexample(ce, other_ce):
+ to_remove.add(other_ce)
+ for other_ce in to_remove:
+ del self.counterexamples[other_ce]
+ self.counterexamples[ce] = np.array([np.zeros(int(b)) for b in self.buckets])
+ return True
+
+ def update_dist_from_multi(self, sample, info, rho):
+ try:
+ iter(rho)
+ except:
+ for i, b in enumerate(info):
+ self.invalid[i][b] += 1.
+ return
+ if len(rho) != self.num_properties:
+ for i, b in enumerate(info):
+ self.invalid[i][b] += 1.
+ return
+
+ counter_ex = tuple(rho[node] < self.thres[node] for node in sorted(self.priority_graph.nodes))
+ error_value = self._compute_error_value(counter_ex)
+ if rulebook.using_continuous:
+ error_value = self._compute_error_value_continuous(rho)
+ print('(dynamic_emab.py) error_value =', error_value)
+ self._update_counterexample(counter_ex)
+ for i, b in enumerate(info):
+ self.counts[i][b] += self.sum_error_weight
+ self.counterexamples[counter_ex][i][b] += error_value
+ self.errors = self._get_total_counterexamples()
+ self.t += 1
+ if self.verbosity >= 2:
+ print('counterexamples =', self.counterexamples)
+ if self.verbosity >= 2:
+ for ce in self.counterexamples:
+ if self._compute_error_value(ce) > 0:
+ print('counterexamples =', ce, ', times =', int(np.sum(self.counterexamples[ce], axis = 1)[0]/self._compute_error_value(ce)))
+ if self.verbosity >= 1:
+ proportions = self.errors / self.counts
+ print('self.errors[0] =', self.errors[0])
+ print('self.counts[0] =', self.counts[0])
+ Q = proportions + np.sqrt(self.exploration_ratio / self.counts * np.log(self.t))
+ print('Q[0] =', Q[0], '\nfirst_term[0] =', proportions[0], '\nsecond_term[0] =', np.sqrt(self.exploration_ratio / self.counts * np.log(self.t))[0], '\nratio[0] =', proportions[0]/(proportions+np.sqrt(self.exploration_ratio / self.counts * np.log(self.t)))[0])
+
+ def _compute_error_value(self, counter_ex):
+ error_value = 0
+ for i in range(len(counter_ex)):
+ error_value += 2**(self.error_weight[i]) * counter_ex[i]
+ return error_value
+
+ def _compute_error_value_continuous(self, rho):
+ error_value = 0
+ for i in range(len(rho)):
+ error_value += 2**(self.error_weight[i]) * -1 * rho[i]
+ return error_value
+
+ def compute_error_weight(self):
+ level = {}
+ for node in nx.topological_sort(self.priority_graph):
+ if self.priority_graph.in_degree(node) == 0:
+ level[node] = 0
+ else:
+ level[node] = max([level[p] for p in self.priority_graph.predecessors(node)]) + 1
+
+ ranking_map = {}
+ ranking_count = {}
+ for rank in sorted(level.values()):
+ if rank not in ranking_count:
+ ranking_count[rank] = 1
+ else:
+ ranking_count[rank] += 1
+ count = 0
+ for key, value in reversed(ranking_count.items()):
+ ranking_map[key] = count
+ count += value
+
+ self.error_weight = {} #node_id -> weight
+ self.sum_error_weight = 0
+ for node in level:
+ if self.priority_graph.nodes[node]['active']:
+ self.error_weight[node] = ranking_map[level[node]]
+ self.sum_error_weight += 2**self.error_weight[node]
+ else:
+ self.error_weight[node] = -1
+ for key, value in sorted(self.error_weight.items()):
+ if self.verbosity >= 2:
+ print(f"Node {key}: {value}")
+
+class DiscreteDynamicEMABSampler(DiscreteCrossEntropySampler):
+ pass
diff --git a/src/verifai/samplers/dynamic_mab.py b/src/verifai/samplers/dynamic_mab.py
new file mode 100644
index 0000000..1d8d8a2
--- /dev/null
+++ b/src/verifai/samplers/dynamic_mab.py
@@ -0,0 +1,244 @@
+import numpy as np
+import networkx as nx
+from itertools import product
+from verifai.samplers.domain_sampler import BoxSampler, DiscreteBoxSampler, \
+ DomainSampler, SplitSampler
+from verifai.samplers.random_sampler import RandomSampler
+from verifai.samplers.cross_entropy import DiscreteCrossEntropySampler
+from verifai.samplers.multi_objective import MultiObjectiveSampler
+from verifai.rulebook import rulebook
+
+class DynamicMultiArmedBanditSampler(DomainSampler):
+ def __init__(self, domain, dmab_params):
+ print('(dynamic_mab.py) Initializing!!!')
+ print('(dynamic_mab.py) dmab_params =', dmab_params)
+ super().__init__(domain)
+ self.alpha = dmab_params.alpha
+ self.thres = dmab_params.thres
+ self.cont_buckets = dmab_params.cont.buckets
+ self.cont_dist = dmab_params.cont.dist
+ self.disc_dist = dmab_params.disc.dist
+ self.cont_ce = lambda domain: ContinuousDynamicMABSampler(domain=domain,
+ buckets=self.cont_buckets,
+ dist=self.cont_dist,
+ alpha=self.alpha,
+ thres=self.thres)
+ self.disc_ce = lambda domain: DiscreteDynamicMABSampler(domain=domain,
+ dist=self.disc_dist,
+ alpha=self.alpha,
+ thres=self.thres)
+ partition = (
+ (lambda d: d.standardizedDimension > 0, self.cont_ce),
+ (lambda d: d.standardizedIntervals, self.disc_ce)
+ )
+ self.split_samplers = {}
+ for id, priority_graph in rulebook.priority_graphs.items():
+ self.split_samplers[id] = SplitSampler.fromPartition(domain,
+ partition,
+ RandomSampler)
+ for subsampler in self.split_samplers[id].samplers:
+ if isinstance(subsampler, ContinuousDynamicMABSampler):
+ print('(dynamic_mab.py) Set priority graph', id)
+ subsampler.set_graph(priority_graph)
+ subsampler.compute_error_weight()
+ elif isinstance(subsampler, DiscreteDynamicMABSampler):
+ assert True
+ else:
+ assert isinstance(subsampler, RandomSampler)
+ node_ids = list(nx.dfs_preorder_nodes(priority_graph))
+ if not sorted(node_ids) == list(range(len(node_ids))):
+ raise ValueError('Node IDs should be in order and start from 0')
+ if not sorted(list(self.split_samplers.keys())) == list(range(len(rulebook.priority_graphs))):
+ raise ValueError('Priority graph IDs should be in order and start from 0')
+ self.num_segs = len(self.split_samplers)
+ print('(dynamic_mab.py) num_segs =', self.num_segs)
+ self.sampler_idx = 0
+ self.using_sampler = rulebook.using_sampler # -1: round-robin
+ assert self.using_sampler < self.num_segs
+ print('(dynamic_mab.py) using_sampler =', self.using_sampler)
+
+ def getSample(self):
+ if self.using_sampler == -1:
+ # Sample from each segment in a round-robin fashion
+ idx = self.sampler_idx % self.num_segs
+ else:
+ idx = self.using_sampler
+ return self.split_samplers[idx].getSample()
+
+ def update(self, sample, info, rhos):
+ # Update each sampler based on the corresponding segment
+ try:
+ iter(rhos)
+ except:
+ for i in range(len(self.split_samplers)):
+ self.split_samplers[i].update(sample, info, rhos)
+ return
+ if self.using_sampler == -1:
+ print('(dynamic_mab.py) Getting feedback from segment', self.sampler_idx % self.num_segs)
+ for i in range(len(rhos)):
+ self.split_samplers[i].update(sample, info, rhos[i])
+ else:
+ print('(dynamic_mab.py) Getting feedback from segment', self.using_sampler)
+ self.split_samplers[self.using_sampler].update(sample, info, rhos[self.using_sampler])
+ self.sampler_idx += 1
+
+class ContinuousDynamicMABSampler(BoxSampler, MultiObjectiveSampler):
+ verbosity = 2
+
+ def __init__(self, domain, alpha, thres,
+ buckets=10, dist=None, restart_every=100):
+ super().__init__(domain)
+ if isinstance(buckets, int):
+ buckets = np.ones(self.dimension) * buckets
+ elif len(buckets) > 1:
+ assert len(buckets) == self.dimension
+ else:
+ buckets = np.ones(self.dimension) * buckets[0]
+ if dist is not None:
+ assert (len(dist) == len(buckets))
+ if dist is None:
+ dist = np.array([np.ones(int(b))/b for b in buckets])
+ self.buckets = buckets # 1*d, each element specifies the number of buckets in that dimension
+ self.dist = dist # N*d, ???
+ self.alpha = alpha
+ self.thres = thres
+ self.current_sample = None
+ self.counts = np.array([np.ones(int(b)) for b in buckets]) # N*d, T (visit times)
+ self.errors = np.array([np.zeros(int(b)) for b in buckets]) # N*d, total times resulting in maximal counterexample
+ self.t = 1 # time, used in Q
+ self.counterexamples = dict()
+ self.is_multi = True #False
+ self.invalid = np.array([np.zeros(int(b)) for b in buckets]) # N*d, ???
+ self.monitor = None
+ self.rho_values = []
+ self.restart_every = restart_every
+ self.exploration_ratio = 2.0
+
+ def getVector(self):
+ return self.generateSample()
+
+ def generateSample(self):
+ proportions = self.errors / self.counts
+ Q = proportions + np.sqrt(self.exploration_ratio / self.counts * np.log(self.t))
+ # choose the bucket with the highest "goodness" value, breaking ties randomly.
+ bucket_samples = np.array([np.random.choice(np.flatnonzero(np.isclose(Q[i], Q[i].max())))
+ for i in range(len(self.buckets))])
+ self.current_sample = bucket_samples
+ ret = tuple(np.random.uniform(bs, bs+1.)/b for b, bs
+ in zip(self.buckets, bucket_samples)) # uniform randomly sample from the range of the bucket
+ return ret, bucket_samples
+
+ def updateVector(self, vector, info, rho):
+ assert rho is not None
+ # "random restarts" to generate a new topological sort of the priority graph
+ # every restart_every samples.
+ if self.is_multi:
+ if self.monitor is not None and self.monitor.linearize and self.t % self.restart_every == 0:
+ self.monitor._linearize()
+ self.update_dist_from_multi(vector, info, rho)
+ return
+ self.t += 1
+ for i, b in enumerate(info):
+ self.counts[i][b] += 1.
+ if rho < self.thres:
+ self.errors[i][b] += 1.
+
+ def is_better_counterexample(self, ce1, ce2):
+ if ce2 is None:
+ return True
+ return self._compute_error_value(ce1) > self._compute_error_value(ce2)
+
+ def _get_total_counterexamples(self):
+ return sum(self.counterexamples.values())
+
+ def _update_counterexample(self, ce, to_delete=False): # update counterexamples, may or may not delete non-maximal counterexamples
+ if ce in self.counterexamples:
+ return True
+ if to_delete:
+ to_remove = set()
+ if len(self.counterexamples) > 0:
+ for other_ce in self.counterexamples:
+ if self.is_better_counterexample(other_ce, ce):
+ return False
+ for other_ce in self.counterexamples:
+ if self.is_better_counterexample(ce, other_ce):
+ to_remove.add(other_ce)
+ for other_ce in to_remove:
+ del self.counterexamples[other_ce]
+ self.counterexamples[ce] = np.array([np.zeros(int(b)) for b in self.buckets])
+ return True
+
+ def update_dist_from_multi(self, sample, info, rho):
+ try:
+ iter(rho)
+ except:
+ for i, b in enumerate(info):
+ self.invalid[i][b] += 1.
+ return
+ if len(rho) != self.num_properties:
+ for i, b in enumerate(info):
+ self.invalid[i][b] += 1.
+ return
+
+ counter_ex = tuple(rho[node] < self.thres[node] for node in sorted(self.priority_graph.nodes))
+ error_value = self._compute_error_value(counter_ex)
+ is_ce = self._update_counterexample(counter_ex, True)
+ for i, b in enumerate(info):
+ self.counts[i][b] += 1
+ if is_ce:
+ self.counterexamples[counter_ex][i][b] += 1
+ self.errors = self._get_total_counterexamples()
+ self.t += 1
+ if self.verbosity >= 2:
+ print('counterexamples =', self.counterexamples)
+ if self.verbosity >= 2:
+ for ce in self.counterexamples:
+ if self._compute_error_value(ce) > 0:
+ print('largest counterexamples =', ce, ', times =', int(np.sum(self.counterexamples[ce], axis = 1)[0]))
+ if self.verbosity >= 1:
+ proportions = self.errors / self.counts
+ print('self.errors[0] =', self.errors[0])
+ print('self.counts[0] =', self.counts[0])
+ Q = proportions + np.sqrt(self.exploration_ratio / self.counts * np.log(self.t))
+ print('Q[0] =', Q[0], '\nfirst_term[0] =', proportions[0], '\nsecond_term[0] =', np.sqrt(2 / self.counts * np.log(self.t))[0], '\nratio[0] =', proportions[0]/(proportions+np.sqrt(2 / self.counts * np.log(self.t)))[0])
+
+ def _compute_error_value(self, counter_ex):
+ error_value = 0
+ for i in range(len(counter_ex)):
+ error_value += 2**(self.error_weight[i]) * counter_ex[i]
+ return error_value
+
+ def compute_error_weight(self):
+ level = {}
+ for node in nx.topological_sort(self.priority_graph):
+ if self.priority_graph.in_degree(node) == 0:
+ level[node] = 0
+ else:
+ level[node] = max([level[p] for p in self.priority_graph.predecessors(node)]) + 1
+
+ ranking_map = {}
+ ranking_count = {}
+ for rank in sorted(level.values()):
+ if rank not in ranking_count:
+ ranking_count[rank] = 1
+ else:
+ ranking_count[rank] += 1
+ count = 0
+ for key, value in reversed(ranking_count.items()):
+ ranking_map[key] = count
+ count += value
+
+ self.error_weight = {} #node_id -> weight
+ self.sum_error_weight = 0
+ for node in level:
+ if self.priority_graph.nodes[node]['active']:
+ self.error_weight[node] = ranking_map[level[node]]
+ self.sum_error_weight += 2**self.error_weight[node]
+ else:
+ self.error_weight[node] = -1
+ for key, value in sorted(self.error_weight.items()):
+ if self.verbosity >= 2:
+ print(f"Node {key}: {value}")
+
+class DiscreteDynamicMABSampler(DiscreteCrossEntropySampler):
+ pass
diff --git a/src/verifai/samplers/dynamic_unified_emab.py b/src/verifai/samplers/dynamic_unified_emab.py
new file mode 100644
index 0000000..73fab0f
--- /dev/null
+++ b/src/verifai/samplers/dynamic_unified_emab.py
@@ -0,0 +1,187 @@
+import numpy as np
+import networkx as nx
+from itertools import product
+from verifai.samplers.domain_sampler import BoxSampler, DiscreteBoxSampler, \
+ DomainSampler, SplitSampler
+from verifai.samplers.random_sampler import RandomSampler
+from verifai.samplers.cross_entropy import DiscreteCrossEntropySampler
+from verifai.samplers.multi_objective import MultiObjectiveSampler
+from verifai.rulebook import rulebook
+
+class DynamicUnifiedExtendedMultiArmedBanditSampler(DomainSampler):
+ def __init__(self, domain, udemab_params):
+ print('(dynamic_unified_emab.py) Initializing!!!')
+ print('(dynamic_unified_emab.py) udemab_params =', udemab_params)
+ super().__init__(domain)
+ self.alpha = udemab_params.alpha
+ self.thres = udemab_params.thres
+ self.cont_buckets = udemab_params.cont.buckets
+ self.cont_dist = udemab_params.cont.dist
+ self.disc_dist = udemab_params.disc.dist
+ self.cont_ce = lambda domain: ContinuousDynamicUnifiedEMABSampler(domain=domain,
+ buckets=self.cont_buckets,
+ dist=self.cont_dist,
+ alpha=self.alpha,
+ thres=self.thres)
+ self.disc_ce = lambda domain: DiscreteDynamicUnifiedEMABSampler(domain=domain,
+ dist=self.disc_dist,
+ alpha=self.alpha,
+ thres=self.thres)
+ partition = (
+ (lambda d: d.standardizedDimension > 0, self.cont_ce),
+ (lambda d: d.standardizedIntervals, self.disc_ce)
+ )
+ self.split_sampler = SplitSampler.fromPartition(domain, partition, RandomSampler)
+
+ def getSample(self):
+ return self.split_sampler.getSample()
+
+ def update(self, sample, info, rhos):
+ # Update each sampler based on the corresponding segment
+ try:
+ iter(rhos)
+ except:
+ self.split_sampler.update(sample, info, rhos)
+ return
+ for subsampler in self.split_sampler.samplers:
+ if isinstance(subsampler, ContinuousDynamicUnifiedEMABSampler):
+ subsampler.set_priority_graphs(rulebook.priority_graphs)
+ self.split_sampler.update(sample, info, rhos)
+
+class ContinuousDynamicUnifiedEMABSampler(BoxSampler, MultiObjectiveSampler):
+ verbosity = 2
+
+ def __init__(self, domain, alpha, thres,
+ buckets=10, dist=None, restart_every=100):
+ super().__init__(domain)
+ if isinstance(buckets, int):
+ buckets = np.ones(self.dimension) * buckets
+ elif len(buckets) > 1:
+ assert len(buckets) == self.dimension
+ else:
+ buckets = np.ones(self.dimension) * buckets[0]
+ if dist is not None:
+ assert (len(dist) == len(buckets))
+ if dist is None:
+ dist = np.array([np.ones(int(b))/b for b in buckets])
+ self.buckets = buckets # 1*d, each element specifies the number of buckets in that dimension
+ self.dist = dist # N*d, ???
+ self.alpha = alpha
+ self.thres = thres
+ self.current_sample = None
+ self.counts = np.array([np.ones(int(b)) for b in buckets]) # N*d, T (visit times)
+ self.errors = np.array([np.zeros(int(b)) for b in buckets]) # N*d, total times resulting in maximal counterexample
+ self.t = 1 # time, used in Q
+ self.is_multi = True #False
+ self.invalid = np.array([np.zeros(int(b)) for b in buckets]) # N*d, ???
+ self.monitor = None
+ self.rho_values = []
+ self.restart_every = restart_every
+
+ def set_priority_graphs(self, graphs):
+ self.priority_graphs = graphs
+ for id, graph in self.priority_graphs.items():
+ node_ids = list(nx.dfs_preorder_nodes(graph))
+ if not sorted(node_ids) == list(range(len(node_ids))):
+ raise ValueError('Node IDs should be in order and start from 0')
+
+ def getVector(self):
+ return self.generateSample()
+
+ def generateSample(self):
+ proportions = self.errors / self.counts
+ Q = proportions + np.sqrt(2 / self.counts * np.log(self.t))
+ # choose the bucket with the highest "goodness" value, breaking ties randomly.
+ bucket_samples = np.array([np.random.choice(np.flatnonzero(np.isclose(Q[i], Q[i].max())))
+ for i in range(len(self.buckets))])
+ self.current_sample = bucket_samples
+ ret = tuple(np.random.uniform(bs, bs+1.)/b for b, bs
+ in zip(self.buckets, bucket_samples)) # uniform randomly sample from the range of the bucket
+ return ret, bucket_samples
+
+ def updateVector(self, vector, info, rhos):
+ assert rhos is not None
+ assert self.is_multi is True
+ if self.is_multi:
+ self.update_dist_from_multi(vector, info, rhos)
+ return
+
+ def update_dist_from_multi(self, sample, info, rhos):
+ try:
+ iter(rhos)
+ except:
+ for i, b in enumerate(info):
+ self.invalid[i][b] += 1.
+ return
+ if len(rhos) != len(self.priority_graphs):
+ for i, b in enumerate(info):
+ self.invalid[i][b] += 1.
+ return
+
+ error_values = []
+ for i, rho in enumerate(rhos):
+ print('Evaluate segment ', i, ' with rho =', rho)
+ assert len(rho) == len(self.priority_graphs[i].nodes)
+ print('sorted(self.priority_graphs[i].nodes) =', sorted(self.priority_graphs[i].nodes))
+ print('self.thres =', self.thres)
+ counter_ex = tuple(rho[node] < self.thres for node in sorted(self.priority_graphs[i].nodes))
+ error_value = self._compute_error_value(counter_ex, i)
+ print('error_value =', error_value)
+ error_values.append(error_value)
+ for i, b in enumerate(info):
+ self.counts[i][b] += 1
+ self.errors[i][b] += sum(error_values) / len(error_values)
+ print('average error_value =', sum(error_values) / len(error_values))
+ self.t += 1
+ if self.verbosity >= 1:
+ proportions = self.errors / self.counts
+ print('self.errors[0] =', self.errors[0])
+ print('self.counts[0] =', self.counts[0])
+ Q = proportions + np.sqrt(2 / self.counts * np.log(self.t))
+ print('Q[0] =', Q[0], '\nfirst_term[0] =', proportions[0], '\nsecond_term[0] =', np.sqrt(2 / self.counts * np.log(self.t))[0], '\nratio[0] =', proportions[0]/(proportions+np.sqrt(2 / self.counts * np.log(self.t)))[0])
+
+ def _compute_error_value(self, counter_ex, graph_idx=None):
+ assert graph_idx is not None
+ self.compute_error_weight(graph_idx)
+ error_value = 0
+ for i in range(len(counter_ex)):
+ error_value += 2**(self.error_weight[i]) * counter_ex[i]
+ return float(error_value/self.sum_error_weight)
+
+ def compute_error_weight(self, graph_idx=None):
+ assert graph_idx is not None
+ self.priority_graph = self.priority_graphs[graph_idx]
+
+ level = {}
+ for node in nx.topological_sort(self.priority_graph):
+ if self.priority_graph.in_degree(node) == 0:
+ level[node] = 0
+ else:
+ level[node] = max([level[p] for p in self.priority_graph.predecessors(node)]) + 1
+
+ ranking_map = {}
+ ranking_count = {}
+ for rank in sorted(level.values()):
+ if rank not in ranking_count:
+ ranking_count[rank] = 1
+ else:
+ ranking_count[rank] += 1
+ count = 0
+ for key, value in reversed(ranking_count.items()):
+ ranking_map[key] = count
+ count += value
+
+ self.error_weight = {} #node_id -> weight
+ self.sum_error_weight = 0
+ for node in level:
+ if self.priority_graph.nodes[node]['active']:
+ self.error_weight[node] = ranking_map[level[node]]
+ self.sum_error_weight += 2**self.error_weight[node]
+ else:
+ self.error_weight[node] = -1
+ for key, value in sorted(self.error_weight.items()):
+ if self.verbosity >= 2:
+ print(f"Node {key}: {value}")
+
+class DiscreteDynamicUnifiedEMABSampler(DiscreteCrossEntropySampler):
+ pass
diff --git a/src/verifai/samplers/extended_multi_armed_bandit.py b/src/verifai/samplers/extended_multi_armed_bandit.py
new file mode 100644
index 0000000..6833ebe
--- /dev/null
+++ b/src/verifai/samplers/extended_multi_armed_bandit.py
@@ -0,0 +1,222 @@
+import numpy as np
+import networkx as nx
+from itertools import product
+from verifai.samplers.domain_sampler import BoxSampler, DiscreteBoxSampler, \
+ DomainSampler, SplitSampler
+from verifai.samplers.random_sampler import RandomSampler
+from verifai.samplers.cross_entropy import DiscreteCrossEntropySampler
+from verifai.samplers.multi_objective import MultiObjectiveSampler
+from verifai.rulebook import rulebook
+
+class ExtendedMultiArmedBanditSampler(DomainSampler):
+ def __init__(self, domain, emab_params):
+ print('(extended_multi_armed_bandit.py) Initializing!!!')
+ print('(extended_multi_armed_bandit.py) emab_params =', emab_params)
+ super().__init__(domain)
+ self.alpha = emab_params.alpha
+ self.thres = emab_params.thres
+ self.cont_buckets = emab_params.cont.buckets
+ self.cont_dist = emab_params.cont.dist
+ self.disc_dist = emab_params.disc.dist
+ self.cont_ce = lambda domain: ContinuousExtendedMultiArmedBanditSampler(domain=domain,
+ buckets=self.cont_buckets,
+ dist=self.cont_dist,
+ alpha=self.alpha,
+ thres=self.thres)
+ self.disc_ce = lambda domain: DiscreteExtendedMultiArmedBanditSampler(domain=domain,
+ dist=self.disc_dist,
+ alpha=self.alpha,
+ thres=self.thres)
+ partition = (
+ (lambda d: d.standardizedDimension > 0, self.cont_ce),
+ (lambda d: d.standardizedIntervals, self.disc_ce)
+ )
+ self.split_sampler = SplitSampler.fromPartition(domain,
+ partition,
+ RandomSampler)
+ self.cont_sampler, self.disc_sampler = None, None
+ self.rand_sampler = None
+ for subsampler in self.split_sampler.samplers:
+ if isinstance(subsampler, ContinuousExtendedMultiArmedBanditSampler):
+ assert self.cont_sampler is None
+ ## TODO: set priority graph here
+ subsampler.set_graph(rulebook.priority_graph)
+ self.cont_sampler = subsampler
+ elif isinstance(subsampler, DiscreteExtendedMultiArmedBanditSampler):
+ assert self.disc_sampler is None
+ self.disc_sampler = subsampler
+ else:
+ assert isinstance(subsampler, RandomSampler)
+ assert self.rand_sampler is None
+ self.rand_sampler = subsampler
+
+ def getSample(self):
+ return self.split_sampler.getSample()
+
+ def update(self, sample, info, rho):
+ self.split_sampler.update(sample, info, rho)
+
+class ContinuousExtendedMultiArmedBanditSampler(BoxSampler, MultiObjectiveSampler):
+ def __init__(self, domain, alpha, thres,
+ buckets=10, dist=None, restart_every=100):
+ super().__init__(domain)
+ if isinstance(buckets, int):
+ buckets = np.ones(self.dimension) * buckets
+ elif len(buckets) > 1:
+ assert len(buckets) == self.dimension
+ else:
+ buckets = np.ones(self.dimension) * buckets[0]
+ if dist is not None:
+ assert (len(dist) == len(buckets))
+ if dist is None:
+ dist = np.array([np.ones(int(b))/b for b in buckets])
+ self.buckets = buckets # 1*d, each element specifies the number of buckets in that dimension
+ self.dist = dist # N*d, ???
+ self.alpha = alpha
+ self.thres = thres
+ self.current_sample = None
+ self.counts = np.array([np.ones(int(b)) for b in buckets]) # N*d, T (visit times)
+ self.errors = np.array([np.zeros(int(b)) for b in buckets]) # N*d, total times resulting in maximal counterexample
+ self.t = 1 # time, used in Q
+ self.counterexamples = dict()
+ self.is_multi = True #False
+ self.invalid = np.array([np.zeros(int(b)) for b in buckets]) # N*d, ???
+ self.monitor = None
+ self.rho_values = []
+ self.restart_every = restart_every
+
+ def getVector(self):
+ return self.generateSample()
+
+ def generateSample(self):
+ proportions = self.errors / self.counts
+ Q = proportions + np.sqrt(2 / self.counts * np.log(self.t))
+ # choose the bucket with the highest "goodness" value, breaking ties randomly.
+ bucket_samples = np.array([np.random.choice(np.flatnonzero(np.isclose(Q[i], Q[i].max())))
+ for i in range(len(self.buckets))])
+ self.current_sample = bucket_samples
+ ret = tuple(np.random.uniform(bs, bs+1.)/b for b, bs
+ in zip(self.buckets, bucket_samples)) # uniform randomly sample from the range of the bucket
+ return ret, bucket_samples
+
+ def updateVector(self, vector, info, rho):
+ assert rho is not None
+ # "random restarts" to generate a new topological sort of the priority graph
+ # every restart_every samples.
+ if self.is_multi:
+ if self.monitor is not None and self.monitor.linearize and self.t % self.restart_every == 0:
+ self.monitor._linearize()
+ self.update_dist_from_multi(vector, info, rho)
+ return
+ self.t += 1
+ for i, b in enumerate(info):
+ self.counts[i][b] += 1.
+ if rho < self.thres:
+ self.errors[i][b] += 1.
+
+ # is rho1 better than rho2?
+ # partial pre-ordering on objective functions, so it is possible that:
+ # is_better_counterexample(rho1, rho2)
+ # and is_better_counterxample(rho2, rho1) both return False
+ def is_better_counterexample(self, ce1, ce2):
+ if ce2 is None:
+ return True
+ all_same = True
+ already_better = [False] * self.num_properties
+ for node in nx.dfs_preorder_nodes(self.priority_graph):
+ if already_better[node]:
+ continue
+ b1 = ce1[node]
+ b2 = ce2[node]
+ all_same = all_same and b1 == b2
+ if b2 and not b1:
+ return False
+ if b1 and not b2:
+ already_better[node] = True
+ for subnode in nx.descendants(self.priority_graph, node):
+ already_better[subnode] = True
+ return not all_same
+
+ def _get_total_counterexamples(self):
+ return sum(self.counterexamples.values())
+
+ @property
+ def counterexample_values(self):
+ return [ce in self.counterexamples for ce in self.rho_values]
+
+ def _add_to_running(self, ce): # update maximal counterexample
+ if ce in self.counterexamples:
+ return True
+ to_remove = set()
+ # if there is already a better counterexample, don't add this.
+ if len(self.counterexamples) > 0:
+ for other_ce in self.counterexamples:
+ if self.is_better_counterexample(other_ce, ce):
+ return False
+ # remove all worse counterexamples than this.
+ for other_ce in self.counterexamples:
+ if self.is_better_counterexample(ce, other_ce):
+ to_remove.add(other_ce)
+ for other_ce in to_remove:
+ del self.counterexamples[other_ce]
+ self.counterexamples[ce] = np.array([np.zeros(int(b)) for b in self.buckets])
+ return True
+
+ def _update_counterexample(self, ce, to_delete=False): # update counterexamples, may or may not delete non-maximal counterexamples
+ if ce in self.counterexamples:
+ return True
+ if to_delete:
+ to_remove = set()
+ if len(self.counterexamples) > 0:
+ for other_ce in self.counterexamples:
+ if self.is_better_counterexample(other_ce, ce):
+ return False
+ for other_ce in self.counterexamples:
+ if self.is_better_counterexample(ce, other_ce):
+ to_remove.add(other_ce)
+ for other_ce in to_remove:
+ del self.counterexamples[other_ce]
+ self.counterexamples[ce] = np.array([np.zeros(int(b)) for b in self.buckets])
+ return True
+
+ def update_dist_from_multi(self, sample, info, rho):
+ try:
+ iter(rho)
+ except:
+ for i, b in enumerate(info):
+ self.invalid[i][b] += 1.
+ return
+ if len(rho) != self.num_properties:
+ for i, b in enumerate(info):
+ self.invalid[i][b] += 1.
+ return
+ counter_ex = tuple(
+ rho[node] < self.thres[node] for node in nx.dfs_preorder_nodes(self.priority_graph)
+ ) # vector of falsification
+ self.rho_values.append(counter_ex)
+ # TODO: generalize
+ error_value = self._compute_error_value(counter_ex)
+ is_ce = self._update_counterexample(counter_ex)
+ for i, b in enumerate(info):
+ self.counts[i][b] += 7.
+ if is_ce:
+ self.counterexamples[counter_ex][i][b] += error_value
+ self.errors = self._get_total_counterexamples()
+ self.t += 1
+ print('counterexamples =', self.counterexamples)
+ for ce in self.counterexamples:
+ if self._compute_error_value(ce) > 0:
+ print('counterexamples =', ce, ', times =', int(np.sum(self.counterexamples[ce], axis = 1)[0]/self._compute_error_value(ce)))
+ proportions = self.errors / self.counts
+ print('self.errors =', self.errors)
+ print('self.counts =', self.counts)
+ Q = proportions + np.sqrt(2 / self.counts * np.log(self.t))
+ print('Q =', Q, '\nfirst_term =', proportions, '\nsecond_term =', np.sqrt(2 / self.counts * np.log(self.t)), '\nratio =', proportions/(proportions+np.sqrt(2 / self.counts * np.log(self.t))))
+
+ def _compute_error_value(self, counter_ex):
+ # TODO: generalize
+ error_value = 4.0*counter_ex[0] + 2.0*counter_ex[1] + 1.0*counter_ex[2]
+ return error_value
+
+class DiscreteExtendedMultiArmedBanditSampler(DiscreteCrossEntropySampler):
+ pass
diff --git a/src/verifai/samplers/feature_sampler.py b/src/verifai/samplers/feature_sampler.py
index 5890505..d65804c 100644
--- a/src/verifai/samplers/feature_sampler.py
+++ b/src/verifai/samplers/feature_sampler.py
@@ -20,6 +20,11 @@
from verifai.samplers.bayesian_optimization import BayesOptSampler
from verifai.samplers.simulated_annealing import SimulatedAnnealingSampler
from verifai.samplers.grid_sampler import GridSampler
+from verifai.samplers.extended_multi_armed_bandit import ExtendedMultiArmedBanditSampler
+from verifai.samplers.dynamic_emab import DynamicExtendedMultiArmedBanditSampler
+from verifai.samplers.dynamic_mab import DynamicMultiArmedBanditSampler
+from verifai.samplers.dynamic_ce import DynamicCrossEntropySampler
+from verifai.samplers.dynamic_unified_emab import DynamicUnifiedExtendedMultiArmedBanditSampler
### Samplers defined over FeatureSpaces
@@ -91,12 +96,89 @@ def multiArmedBanditSamplerFor(space, mab_params=None):
Uses random sampling for lengths of feature lists and any Domains
that are not standardizable.
"""
+ print('(feature_sampler.py) Using mab sampler')
if mab_params is None:
mab_params = default_sampler_params('mab')
+ print('(feature_sampler.py) mab_params =', mab_params)
return LateFeatureSampler(space, RandomSampler,
lambda domain: MultiArmedBanditSampler(domain=domain,
mab_params=mab_params))
+ @staticmethod
+ def extendedMultiArmedBanditSamplerFor(space, emab_params=None):
+ """Creates an extended multi-armed bandit sampler for a given space.
+
+ Uses random sampling for lengths of feature lists and any Domains
+ that are not standardizable.
+ """
+ print('(feature_sampler.py) Using emab sampler')
+ if emab_params is None:
+ emab_params = default_sampler_params('emab')
+ print('(feature_sampler.py) emab_params =', emab_params)
+ return LateFeatureSampler(space, RandomSampler,
+ lambda domain: ExtendedMultiArmedBanditSampler(domain=domain,
+ emab_params=emab_params))
+
+ @staticmethod
+ def dynamicExtendedMultiArmedBanditSamplerFor(space, demab_params=None):
+ """Creates a dynamic extended multi-armed bandit sampler for a given space.
+
+ Uses random sampling for lengths of feature lists and any Domains
+ that are not standardizable.
+ """
+ print('(feature_sampler.py) Using demab sampler')
+ if demab_params is None:
+ demab_params = default_sampler_params('demab')
+ print('(feature_sampler.py) demab_params =', demab_params)
+ return LateFeatureSampler(space, RandomSampler,
+ lambda domain: DynamicExtendedMultiArmedBanditSampler(domain=domain,
+ demab_params=demab_params))
+
+ @staticmethod
+ def dynamicMultiArmedBanditSamplerFor(space, dmab_params=None):
+ """Creates a dynamic multi-armed bandit sampler for a given space.
+
+ Uses random sampling for lengths of feature lists and any Domains
+ that are not standardizable.
+ """
+ print('(feature_sampler.py) Using dmab sampler')
+ if dmab_params is None:
+ dmab_params = default_sampler_params('dmab')
+ print('(feature_sampler.py) dmab_params =', dmab_params)
+ return LateFeatureSampler(space, RandomSampler,
+ lambda domain: DynamicMultiArmedBanditSampler(domain=domain,
+ dmab_params=dmab_params))
+
+ @staticmethod
+ def dynamicCrossEntropySamplerFor(space, dce_params=None):
+ """Creates a dynamic cross-entropy sampler for a given space.
+
+ Uses random sampling for lengths of feature lists and any Domains
+ that are not standardizable.
+ """
+ print('(feature_sampler.py) Using dce sampler')
+ if dce_params is None:
+ dce_params = default_sampler_params('dce')
+ print('(feature_sampler.py) dce_params =', dce_params)
+ return LateFeatureSampler(space, RandomSampler,
+ lambda domain: DynamicCrossEntropySampler(domain=domain,
+ dce_params=dce_params))
+
+ @staticmethod
+ def dynamicUnifiedExtendedMultiArmedBanditSamplerFor(space, udemab_params=None):
+ """Creates a dynamic unified extended multi-armed bandit sampler for a given space.
+
+ Uses random sampling for lengths of feature lists and any Domains
+ that are not standardizable.
+ """
+ print('(feature_sampler.py) Using udemab sampler')
+ if udemab_params is None:
+ udemab_params = default_sampler_params('udemab')
+ print('(feature_sampler.py) udemab_params =', udemab_params)
+ return LateFeatureSampler(space, RandomSampler,
+ lambda domain: DynamicUnifiedExtendedMultiArmedBanditSampler(domain=domain,
+ udemab_params=udemab_params))
+
@staticmethod
def gridSamplerFor(space, grid_params=None):
"""Creates a grid sampler for a given space.
@@ -258,7 +340,11 @@ def makeRandomSampler(domain):
def default_sampler_params(sampler_type):
if sampler_type == 'halton':
return DotMap(sample_index=0, bases_skipped=0)
- elif sampler_type in ('ce', 'eg', 'mab'):
+ elif sampler_type in ('ce', 'eg', 'mab', 'emab'):
+ cont = DotMap(buckets=5, dist=None)
+ disc = DotMap(dist=None)
+ return DotMap(alpha=0.9, thres=0.0, cont=cont, disc=disc)
+ elif sampler_type in ('demab', 'dmab', 'dce', 'udemab'):
cont = DotMap(buckets=5, dist=None)
disc = DotMap(dist=None)
return DotMap(alpha=0.9, thres=0.0, cont=cont, disc=disc)
diff --git a/src/verifai/samplers/multi_armed_bandit.py b/src/verifai/samplers/multi_armed_bandit.py
index a6c6e39..ebebb64 100644
--- a/src/verifai/samplers/multi_armed_bandit.py
+++ b/src/verifai/samplers/multi_armed_bandit.py
@@ -6,9 +6,12 @@
from verifai.samplers.random_sampler import RandomSampler
from verifai.samplers.cross_entropy import DiscreteCrossEntropySampler
from verifai.samplers.multi_objective import MultiObjectiveSampler
+from verifai.rulebook import rulebook
class MultiArmedBanditSampler(DomainSampler):
def __init__(self, domain, mab_params):
+ print('(multi_armed_bandit.py) Initializing!!!')
+ print('(multi_armed_bandit.py) mab_params =', mab_params)
super().__init__(domain)
self.alpha = mab_params.alpha
self.thres = mab_params.thres
@@ -36,8 +39,10 @@ def __init__(self, domain, mab_params):
for subsampler in self.split_sampler.samplers:
if isinstance(subsampler, ContinuousMultiArmedBanditSampler):
assert self.cont_sampler is None
- if 'priority_graph' in mab_params:
- subsampler.set_graph(mab_params.priority_graph)
+ ## TODO: set priority graph here
+ subsampler.set_graph(rulebook.priority_graph)
+ #if 'priority_graph' in mab_params:
+ # subsampler.set_graph(mab_params.priority_graph)
self.cont_sampler = subsampler
elif isinstance(subsampler, DiscreteMultiArmedBanditSampler):
assert self.disc_sampler is None
@@ -67,38 +72,38 @@ def __init__(self, domain, alpha, thres,
assert (len(dist) == len(buckets))
if dist is None:
dist = np.array([np.ones(int(b))/b for b in buckets])
- self.buckets = buckets
- self.dist = dist
+ self.buckets = buckets # 1*d, each element specifies the number of buckets in that dimension
+ self.dist = dist # N*d
self.alpha = alpha
self.thres = thres
self.current_sample = None
- self.counts = np.array([np.ones(int(b)) for b in buckets])
- self.errors = np.array([np.zeros(int(b)) for b in buckets])
- self.t = 1
+ self.counts = np.array([np.ones(int(b)) for b in buckets]) # N*d, T (visit times)
+ self.errors = np.array([np.zeros(int(b)) for b in buckets]) # N*d, total times resulting in maximal counterexample
+ self.t = 1 # time, used in Q
self.counterexamples = dict()
- self.is_multi = False
- self.invalid = np.array([np.zeros(int(b)) for b in buckets])
+ self.is_multi = True #False
+ self.invalid = np.array([np.zeros(int(b)) for b in buckets]) # N*d
self.monitor = None
self.rho_values = []
self.restart_every = restart_every
+ self.exploration_ratio = 8.0
def getVector(self):
return self.generateSample()
def generateSample(self):
proportions = self.errors / self.counts
- Q = proportions + np.sqrt(2 / self.counts * np.log(self.t))
+ Q = proportions + np.sqrt(self.exploration_ratio / self.counts * np.log(self.t))
# choose the bucket with the highest "goodness" value, breaking ties randomly.
bucket_samples = np.array([np.random.choice(np.flatnonzero(np.isclose(Q[i], Q[i].max())))
for i in range(len(self.buckets))])
self.current_sample = bucket_samples
ret = tuple(np.random.uniform(bs, bs+1.)/b for b, bs
- in zip(self.buckets, bucket_samples))
+ in zip(self.buckets, bucket_samples)) # uniform randomly sample from the range of the bucket
return ret, bucket_samples
def updateVector(self, vector, info, rho):
assert rho is not None
- self.t += 1
# "random restarts" to generate a new topological sort of the priority graph
# every restart_every samples.
if self.is_multi:
@@ -106,6 +111,7 @@ def updateVector(self, vector, info, rho):
self.monitor._linearize()
self.update_dist_from_multi(vector, info, rho)
return
+ self.t += 1
for i, b in enumerate(info):
self.counts[i][b] += 1.
if rho < self.thres:
@@ -141,7 +147,7 @@ def _get_total_counterexamples(self):
def counterexample_values(self):
return [ce in self.counterexamples for ce in self.rho_values]
- def _add_to_running(self, ce):
+ def _add_to_running(self, ce): # update maximal counterexample
if ce in self.counterexamples:
return True
to_remove = set()
@@ -170,19 +176,24 @@ def update_dist_from_multi(self, sample, info, rho):
for i, b in enumerate(info):
self.invalid[i][b] += 1.
return
- # print('inside update_dist_from_multi')
counter_ex = tuple(
rho[node] < self.thres[node] for node in nx.dfs_preorder_nodes(self.priority_graph)
- )
+ ) # vector of falsification
self.rho_values.append(counter_ex)
- # print(f'counter_ex = {counter_ex}')
- # print(self.counterexamples)
is_ce = self._add_to_running(counter_ex)
for i, b in enumerate(info):
self.counts[i][b] += 1.
if is_ce:
self.counterexamples[counter_ex][i][b] += 1.
- self.errors = self.invalid + self._get_total_counterexamples()
+ #self.errors = self.invalid + self._get_total_counterexamples()
+ self.errors = self._get_total_counterexamples()
+ self.t += 1
+ print('counterexamples =', self.counterexamples)
+ for ce in self.counterexamples:
+ print('largest counterexamples =', ce, ', times =', int(np.sum(self.counterexamples[ce], axis = 1)[0]))
+ proportions = self.errors / self.counts
+ Q = proportions + np.sqrt(2 / self.counts * np.log(self.t))
+ print('Q =', Q, '\nfirst_term =', proportions, '\nsecond_term =', np.sqrt(self.exploration_ratio / self.counts * np.log(self.t)), '\nratio =', proportions/(proportions+np.sqrt(2 / self.counts * np.log(self.t))))
class DiscreteMultiArmedBanditSampler(DiscreteCrossEntropySampler):
pass
\ No newline at end of file
diff --git a/src/verifai/scenic_server.py b/src/verifai/scenic_server.py
index fc58dcd..9c75b5b 100644
--- a/src/verifai/scenic_server.py
+++ b/src/verifai/scenic_server.py
@@ -51,11 +51,18 @@ def __init__(self, sampling_data, monitor, options={}):
self.simulator = self.sampler.scenario.getSimulator()
else:
self.simulator = defaults.simulator
+ self.dynamic = defaults.get('dynamic', False)
def evaluate_sample(self, sample):
scene = self.sampler.lastScene
assert scene
result = self._simulate(scene)
+ if self.dynamic:
+ while result is None:
+ sample = self.get_sample(1)
+ scene = self.sampler.lastScene
+ assert scene
+ result = self._simulate(scene)
if result is None:
return self.rejectionFeedback
value = (0 if self.monitor is None
diff --git a/src/verifai/server.py b/src/verifai/server.py
index ef4b043..c4b246c 100644
--- a/src/verifai/server.py
+++ b/src/verifai/server.py
@@ -46,6 +46,8 @@ def choose_sampler(sample_space, sampler_type,
sample_space, ce_params=ce_params)
return 'ce', sampler
if sampler_type == 'mab':
+ print('(server.py) Choosing mab sampler')
+ print('(server.py) sampler_params =', sampler_params)
if sampler_params is None:
mab_params = default_sampler_params('mab')
else:
@@ -66,6 +68,117 @@ def choose_sampler(sample_space, sampler_type,
sampler = FeatureSampler.multiArmedBanditSamplerFor(
sample_space, mab_params=mab_params)
return 'mab', sampler
+ if sampler_type == 'emab':
+ print('(server.py) Choosing emab sampler')
+ print('(server.py) sampler_params =', sampler_params)
+ if sampler_params is None:
+ emab_params = default_sampler_params('emab')
+ else:
+ emab_params = default_sampler_params('emab')
+ if 'cont' in sampler_params:
+ if 'buckets' in sampler_params.cont:
+ emab_params.cont.buckets = sampler_params.cont.buckets
+ if 'dist' in sampler_params.cont:
+ emab_params.cont.dist = sampler_params.cont.dist
+ if 'dist' in sampler_params.disc:
+ emab_params.disc.dist = sampler_params.disc.dist
+ if 'alpha' in sampler_params:
+ emab_params.alpha = sampler_params.alpha
+ if 'thres' in sampler_params:
+ emab_params.thres = sampler_params.thres
+ if 'priority_graph' in sampler_params:
+ emab_params.priority_graph = sampler_params.priority_graph
+ sampler = FeatureSampler.extendedMultiArmedBanditSamplerFor(
+ sample_space, emab_params=emab_params)
+ return 'emab', sampler
+ if sampler_type == 'demab':
+ print('(server.py) Choosing demab sampler')
+ print('(server.py) sampler_params =', sampler_params)
+ if sampler_params is None:
+ demab_params = default_sampler_params('demab')
+ else:
+ demab_params = default_sampler_params('demab')
+ if 'cont' in sampler_params:
+ if 'buckets' in sampler_params.cont:
+ demab_params.cont.buckets = sampler_params.cont.buckets
+ if 'dist' in sampler_params.cont:
+ demab_params.cont.dist = sampler_params.cont.dist
+ if 'dist' in sampler_params.disc:
+ demab_params.disc.dist = sampler_params.disc.dist
+ if 'alpha' in sampler_params:
+ demab_params.alpha = sampler_params.alpha
+ if 'thres' in sampler_params:
+ demab_params.thres = sampler_params.thres
+ if 'priority_graph' in sampler_params:
+ demab_params.priority_graph = sampler_params.priority_graph
+ sampler = FeatureSampler.dynamicExtendedMultiArmedBanditSamplerFor(
+ sample_space, demab_params=demab_params)
+ return 'demab', sampler
+ if sampler_type == 'dmab':
+ print('(server.py) Choosing dmab sampler')
+ print('(server.py) sampler_params =', sampler_params)
+ if sampler_params is None:
+ dmab_params = default_sampler_params('dmab')
+ else:
+ dmab_params = default_sampler_params('dmab')
+ if 'cont' in sampler_params:
+ if 'buckets' in sampler_params.cont:
+ dmab_params.cont.buckets = sampler_params.cont.buckets
+ if 'dist' in sampler_params.cont:
+ dmab_params.cont.dist = sampler_params.cont.dist
+ if 'dist' in sampler_params.disc:
+ dmab_params.disc.dist = sampler_params.disc.dist
+ if 'alpha' in sampler_params:
+ dmab_params.alpha = sampler_params.alpha
+ if 'thres' in sampler_params:
+ dmab_params.thres = sampler_params.thres
+ if 'priority_graph' in sampler_params:
+ dmab_params.priority_graph = sampler_params.priority_graph
+ sampler = FeatureSampler.dynamicMultiArmedBanditSamplerFor(
+ sample_space, dmab_params=dmab_params)
+ return 'dmab', sampler
+ if sampler_type == 'dce':
+ print('(server.py) Choosing dce sampler')
+ print('(server.py) sampler_params =', sampler_params)
+ if sampler_params is None:
+ dce_params = default_sampler_params('dce')
+ else:
+ dce_params = default_sampler_params('dce')
+ if 'cont' in sampler_params:
+ if 'buckets' in sampler_params.cont:
+ dce_params.cont.buckets = sampler_params.cont.buckets
+ if 'dist' in sampler_params.cont:
+ dce_params.cont.dist = sampler_params.cont.dist
+ if 'dist' in sampler_params.disc:
+ dce_params.disc.dist = sampler_params.disc.dist
+ if 'alpha' in sampler_params:
+ dce_params.alpha = sampler_params.alpha
+ if 'thres' in sampler_params:
+ dce_params.thres = sampler_params.thres
+ sampler = FeatureSampler.dynamicCrossEntropySamplerFor(
+ sample_space, dce_params=dce_params)
+ return 'dce', sampler
+ if sampler_type == 'udemab':
+ print('(server.py) Choosing udemab sampler')
+ print('(server.py) sampler_params =', sampler_params)
+ if sampler_params is None:
+ udemab_params = default_sampler_params('udemab')
+ else:
+ udemab_params = default_sampler_params('udemab')
+ if 'cont' in sampler_params:
+ if 'buckets' in sampler_params.cont:
+ udemab_params.cont.buckets = sampler_params.cont.buckets
+ if 'dist' in sampler_params.cont:
+ udemab_params.cont.dist = sampler_params.cont.dist
+ if 'dist' in sampler_params.disc:
+ udemab_params.disc.dist = sampler_params.disc.dist
+ if 'alpha' in sampler_params:
+ udemab_params.alpha = sampler_params.alpha
+ if 'thres' in sampler_params:
+ udemab_params.thres = sampler_params.thres
+ sampler = FeatureSampler.dynamicUnifiedExtendedMultiArmedBanditSamplerFor(
+ sample_space, udemab_params=udemab_params)
+ return 'udemab', sampler
if sampler_type == 'eg':
if sampler_params is None:
eg_params = default_sampler_params('eg')