Missing dynamic layout features (#1838)

* Fix Zero segment location.

* Fixed has_zero_segment naming

* Fix prover input.

* Fixed version reading when no version is supplied

* Added change to changelog.

* fix test_from_serializable()

* fix panic_impl error

* fix cairo version

* Add dummy changelog

* Pin wasm-bindgen

* Register change in CHANGELOG

* Update Cargo.lock

* Remove changes from CHANGELOG

* Add argument parsing for layout params file

* Add dynamic support (no implement)

* Add cairo_layout_params_file.example.json

* Implement dynamic layout creation

* Update CHANGELOG

* Add cli dynamic support for cairo 1

* Make wasm compatible

* Use public_memory_fraction = 4 vy default

* Deserialize bool from int

* Add comparison with python-vm (failing)

* Rebuild .rs files in makefile

* Use 8 as dynamic public_memory_fraction

The same value is used in python-vm

* Use None ratio for dynamic unused builtins

* Add rangecheck96 to private inputs

* Make dyn py files depend on params_file

* Use cpu_component_step=1 by default

* Fix typo in private inputs

* Add range check value to air private input test

* Fix zero segment location

* Use zero builtin instead of None

* Add debug scripts

* Remove dup makefile recipes

* remove outdated test

* Enable ensure-no_std on test

* Fix tests

* Add correct test

* Rename tset

* Add comment

* Add debugging document

* Update cairo layout params file

* Remove duplicated range check

* Remove dup

* Add support for dynamic memory units per step

* Add changelog

* Add support for negative log units per step

* Add LowRatio struct to RangeCheck and Mod builtins

* Fix test

* Use ratio_den in get_allocated_instances

* Remove debugging and scrippts (moveed to another branch)

* Add comment

* Add tests

* Add dynamic test to cairo-vm-cli

* Add parse test

* Remove compare all dynamic

* Add script for comparing with dynamic layouts

* Add tests to workflow

* Delete logic changes

They are going to be moved to another branch

* Delete more logic changes

* Reorder fields

* Remove ref

* Add modulo tests with all_cairo

* Add tests

* Update rust.yml

* Rename compare_outputs_dynamic_layout.sh to compare_outputs_dynamic_layouts.sh

* Update test script

* Remove tests

* Enforce prover constraints in add, sub, and mul

* Remove debug prints

* Add tests

* Update CHANGELOG.md

* Fix serialization

* Comment failing test

* Uncomment test

* Add fractional units per step test

* Change safe_div for div_ceil (bug)

* Add ratio_den tests

* Fix tests

* Remove operation struct and use builtin type instead

* Add unit tests to modulo builtin operations

* Fix security error message

* Add mod_builtin to coverage

* Test custom serde impl

* Upload mod_builtin coverage

* Test custom serde impl

* Upload mod_builtin coverage

* Add Debug scripts (#1839)

* Add debug scripts

* Add debugging document

* Add changelog

* Revert changelog

* Improve script usage code blocks

* Remove extra space

* Add cpu_component_step

* Add comments

* Add cairo pie tests

* Restore zip

* Use .rs.pie.zip

* Update Changelog

* Add dynamic layout documentation

* Update CHANGELOG.md

---------

Co-authored-by: Alon Titelman <[email protected]>
Co-authored-by: Yuval Goldberg <[email protected]>
Co-authored-by: Omri Eshhar <[email protected]>
Co-authored-by: Pedro Fontana <[email protected]>

Author: 5 people

.github/workflows/rust.yml

@@ -757,7 +757,7 @@ jobs:
 
   compare-outputs-dynamic-layouts:
     name: Compare outputs with dynamic layouts
-    needs: [ build-programs, build-release ]
+    needs: [ build-programs, build-release, run-cairo-release ]
     runs-on: ubuntu-22.04
     steps:
     - name: Checkout
@@ -791,6 +791,18 @@ jobs:
         key: cairo_proof_programs-cache-${{ hashFiles('cairo_programs/**/*.cairo', 'examples/wasm-demo/src/array_sum.cairo') }}
         fail-on-cache-miss: true
 
+    - name: Fetch pie
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          cairo_programs/**/*.memory
+          cairo_programs/**/*.trace
+          cairo_programs/**/*.air_public_input
+          cairo_programs/**/*.air_private_input
+          cairo_programs/**/*.pie.zip
+        key: cairo_test_programs-release-trace-cache-${{ github.sha }}
+        fail-on-cache-miss: true
+
     - name: Run script
       run: ./vm/src/tests/compare_outputs_dynamic_layouts.sh
 

CHANGELOG.md

@@ -2,14 +2,14 @@
 
 #### Upcoming Changes
 
-* fix: [#1841](https://github.com/lambdaclass/cairo-vm/pull/1841):
-  * Fix modulo builtin to comply with prover constraints
-
-* feat(BREAKING): [#1824](https://github.com/lambdaclass/cairo-vm/pull/1824):
+* feat(BREAKING): [#1824](https://github.com/lambdaclass/cairo-vm/pull/1824)[#1838](https://github.com/lambdaclass/cairo-vm/pull/1838):
     * Add support for dynamic layout
     * CLI change(BREAKING): The flag `cairo_layout_params_file` must be specified when using dynamic layout.
     * Signature change(BREAKING): Both `CairoRunner::new` and `CairoRunner::new_v2` now receive an `Option<CairoLayoutParams>`, used only with dynamic layout.
 
+* fix: [#1841](https://github.com/lambdaclass/cairo-vm/pull/1841):
+  * Fix modulo builtin to comply with prover constraints
+
 * chore: bump pip `cairo-lang` 0.13.2 [#1827](https://github.com/lambdaclass/cairo-vm/pull/1827)
 
 * chore: bump `cairo-lang-` dependencies to 2.8.0 [#1833](https://github.com/lambdaclass/cairo-vm/pull/1833/files)

README.md

@@ -266,6 +266,15 @@ Now that you have the dependencies necessary to run the test suite you can run:
 make test
 ```
 
+### Using a Dynamic Layout
+
+A dynamic layout must be specified with a dynamic params file. You can find an example in: `vm/src/tests/cairo_layout_params_file.json`.
+
+To run cairo 0 or 1 programs with a dynamic layout, you must use `--layout dynamic` and the `--cairo_layout_params_file` flag pointing a dynamic params file. For example, run:
+```bash
+cargo run --bin cairo-vm-cli cairo_programs/fibonacci.json --layout dynamic --cairo_layout_params_file vm/src/tests/cairo_layout_params_file.json
+```
+
 ### Tracer
 
 Cairo-vm offers a tracer which gives you a visualization of how your memory and registers change line after line as the VM executes the code. You can read more about it [here](./docs/tracer/README.md)

vm/src/tests/compare_outputs_dynamic_layouts.sh

@@ -68,6 +68,70 @@ cat <<EOF > "$TEMP_FOLDER/double_all_cairo.json"
 }
 EOF
 
+cat <<EOF > "$TEMP_FOLDER/fractional_units_per_step.json"
+{
+    "rc_units": 4,
+    "log_diluted_units_per_step": -2,
+    "cpu_component_step": 8,
+    "memory_units_per_step": 8,
+    "uses_pedersen_builtin": false,
+    "pedersen_ratio": 0,
+    "uses_range_check_builtin": false,
+    "range_check_ratio": 0,
+    "uses_ecdsa_builtin": false,
+    "ecdsa_ratio": 0,
+    "uses_bitwise_builtin": false,
+    "bitwise_ratio": 0,
+    "uses_ec_op_builtin": false,
+    "ec_op_ratio": 0,
+    "uses_keccak_builtin": false,
+    "keccak_ratio": 0,
+    "uses_poseidon_builtin": false,
+    "poseidon_ratio": 0,
+    "uses_range_check96_builtin": false,
+    "range_check96_ratio": 0,
+    "range_check96_ratio_den": 1,
+    "uses_add_mod_builtin": false,
+    "add_mod_ratio": 0,
+    "add_mod_ratio_den": 1,
+    "uses_mul_mod_builtin": false,
+    "mul_mod_ratio": 0,
+    "mul_mod_ratio_den": 1
+}
+EOF
+
+cat <<EOF > "$TEMP_FOLDER/ratio_den.json"
+{
+    "rc_units": 4,
+    "log_diluted_units_per_step": 4,
+    "cpu_component_step": 8,
+    "memory_units_per_step": 512,
+    "uses_pedersen_builtin": false,
+    "pedersen_ratio": 0,
+    "uses_range_check_builtin": false,
+    "range_check_ratio": 0,
+    "uses_ecdsa_builtin": false,
+    "ecdsa_ratio": 0,
+    "uses_bitwise_builtin": false,
+    "bitwise_ratio": 0,
+    "uses_ec_op_builtin": false,
+    "ec_op_ratio": 0,
+    "uses_keccak_builtin": false,
+    "keccak_ratio": 0,
+    "uses_poseidon_builtin": false,
+    "poseidon_ratio": 0,
+    "uses_range_check96_builtin": true,
+    "range_check96_ratio": 1,
+    "range_check96_ratio_den": 2,
+    "uses_add_mod_builtin": true,
+    "add_mod_ratio": 1,
+    "add_mod_ratio_den": 2,
+    "uses_mul_mod_builtin": true,
+    "mul_mod_ratio": 1,
+    "mul_mod_ratio_den": 2
+}
+EOF
+
 # Build cases to execute
 CASES=(
     "cairo_programs/proof_programs/factorial.json;all_cairo"
@@ -82,9 +146,33 @@ CASES=(
     "cairo_programs/proof_programs/sha256.json;double_all_cairo"
     "cairo_programs/proof_programs/keccak.json;all_cairo"
     "cairo_programs/proof_programs/keccak.json;double_all_cairo"
+    # Mod builtin feature
     "cairo_programs/mod_builtin_feature/proof/mod_builtin.json;all_cairo"
     "cairo_programs/mod_builtin_feature/proof/mod_builtin_failure.json;all_cairo"
     "cairo_programs/mod_builtin_feature/proof/apply_poly.json;all_cairo"
+    # Fractional units per step
+    "cairo_programs/proof_programs/factorial.json;fractional_units_per_step"
+    "cairo_programs/proof_programs/fibonacci.json;fractional_units_per_step"
+    # Ratio den
+    "cairo_programs/mod_builtin_feature/proof/mod_builtin.json;ratio_den"
+    "cairo_programs/mod_builtin_feature/proof/mod_builtin_failure.json;ratio_den"
+    "cairo_programs/mod_builtin_feature/proof/apply_poly.json;ratio_den"
+)
+
+# Build pie cases to execute
+PIE_CASES=(
+    "cairo_programs/fibonacci.rs.pie.zip;all_cairo"
+    "cairo_programs/fibonacci.rs.pie.zip;double_all_cairo"
+    "cairo_programs/factorial.rs.pie.zip;all_cairo"
+    "cairo_programs/factorial.rs.pie.zip;double_all_cairo"
+    "cairo_programs/bigint.rs.pie.zip;all_cairo"
+    "cairo_programs/bigint.rs.pie.zip;double_all_cairo"
+    "cairo_programs/dict.rs.pie.zip;all_cairo"
+    "cairo_programs/dict.rs.pie.zip;double_all_cairo"
+    "cairo_programs/sha256.rs.pie.zip;all_cairo"
+    "cairo_programs/sha256.rs.pie.zip;double_all_cairo"
+    "cairo_programs/keccak.rs.pie.zip;all_cairo"
+    "cairo_programs/keccak.rs.pie.zip;double_all_cairo"
 )
 
 passed_tests=0
@@ -155,6 +243,51 @@ for case in "${CASES[@]}"; do
     rm program_py.*
 done
 
+
+for case in "${PIE_CASES[@]}"; do
+    IFS=";" read -r program layout <<< "$case"
+
+    full_program="$program"
+    full_layout="$TEMP_FOLDER/$layout.json"
+
+    # Run cairo-vm
+    echo "Running cairo-vm with case: $case"
+    cargo run -p cairo-vm-cli --features mod_builtin --release -- "$full_program" \
+        --layout "dynamic" --cairo_layout_params_file "$full_layout" --run_from_cairo_pie  \
+        --trace_file program_rs.trace --memory_file program_rs.memory
+
+    # Run cairo-lang
+    echo "Running cairo-lang with case: $case"
+    cairo-run --run_from_cairo_pie "$full_program" \
+        --layout "dynamic" --cairo_layout_params_file "$full_layout" \
+        --trace_file program_py.trace --memory_file program_py.memory
+
+    # Compare trace
+    echo "Running trace comparison for case: $case"
+    if ! diff -q program_rs.trace program_py.trace; then
+        echo "Trace differs for case: $case"
+        exit_code=1
+        failed_tests=$((failed_tests + 1))
+    else
+        passed_tests=$((passed_tests + 1))
+    fi
+
+    # Compare memory
+    echo "Running memory comparison for case: $case"
+    if ! ./vm/src/tests/memory_comparator.py program_rs.memory program_py.memory; then
+        echo "Memory differs for case: $case"
+        exit_code=1
+        failed_tests=$((failed_tests + 1))
+    else
+        passed_tests=$((passed_tests + 1))
+    fi
+
+    # Clean files generated by the script
+    echo "Cleaning files"
+    rm program_rs.*
+    rm program_py.*
+done
+
 if test $failed_tests != 0; then
     echo "Comparisons: $failed_tests failed, $passed_tests passed, $((failed_tests + passed_tests)) total"
 elif test $passed_tests = 0; then

vm/src/types/instance_definitions/builtins_instance_def.rs

@@ -1,6 +1,7 @@
 use crate::types::layout::CairoLayoutParams;
 
 use super::mod_instance_def::ModInstanceDef;
+use super::LowRatio;
 use super::{
     bitwise_instance_def::BitwiseInstanceDef, ec_op_instance_def::EcOpInstanceDef,
     ecdsa_instance_def::EcdsaInstanceDef, keccak_instance_def::KeccakInstanceDef,
@@ -199,7 +200,7 @@ impl BuiltinsInstanceDef {
             ratio: Some(params.pedersen_ratio),
         });
         let range_check = Some(RangeCheckInstanceDef {
-            ratio: Some(params.range_check_ratio),
+            ratio: Some(LowRatio::new_int(params.range_check_ratio)),
         });
         let ecdsa = Some(EcdsaInstanceDef {
             ratio: Some(params.ecdsa_ratio),
@@ -217,17 +218,26 @@ impl BuiltinsInstanceDef {
             ratio: Some(params.poseidon_ratio),
         });
         let range_check96 = Some(RangeCheckInstanceDef {
-            ratio: Some(params.range_check96_ratio),
+            ratio: Some(LowRatio::new(
+                params.range_check96_ratio,
+                params.range_check96_ratio_den,
+            )),
         });
         #[cfg(feature = "mod_builtin")]
         let add_mod = Some(ModInstanceDef {
-            ratio: Some(params.add_mod_ratio),
+            ratio: Some(LowRatio::new(
+                params.add_mod_ratio,
+                params.add_mod_ratio_den,
+            )),
             word_bit_len: 96,
             batch_size: 1,
         });
         #[cfg(feature = "mod_builtin")]
         let mul_mod = Some(ModInstanceDef {
-            ratio: Some(params.mul_mod_ratio),
+            ratio: Some(LowRatio::new(
+                params.mul_mod_ratio,
+                params.mul_mod_ratio_den,
+            )),
             word_bit_len: 96,
             batch_size: 1,
         });

vm/src/types/instance_definitions/diluted_pool_instance_def.rs

@@ -3,6 +3,7 @@ use serde::Serialize;
 #[derive(Serialize, Debug, PartialEq)]
 pub(crate) struct DilutedPoolInstanceDef {
     pub(crate) units_per_step: u32, // 2 ^ log_units_per_step (for cairo_lang comparison)
+    pub(crate) fractional_units_per_step: bool, // true when log_units_per_step is negative
     pub(crate) spacing: u32,
     pub(crate) n_bits: u32,
 }
@@ -11,6 +12,7 @@ impl DilutedPoolInstanceDef {
     pub(crate) fn default() -> Self {
         DilutedPoolInstanceDef {
             units_per_step: 16,
+            fractional_units_per_step: false,
             spacing: 4,
             n_bits: 16,
         }
@@ -21,6 +23,15 @@ impl DilutedPoolInstanceDef {
             units_per_step,
             spacing,
             n_bits,
+            ..Self::default()
+        }
+    }
+
+    pub(crate) fn from_log_units_per_step(log_units_per_step: i32) -> Self {
+        DilutedPoolInstanceDef {
+            units_per_step: 2_u32.pow(log_units_per_step.unsigned_abs()),
+            fractional_units_per_step: log_units_per_step.is_negative(),
+            ..DilutedPoolInstanceDef::default()
         }
     }
 }

vm/src/types/instance_definitions/mod.rs

@@ -1,3 +1,5 @@
+use serde::Serialize;
+
 pub mod bitwise_instance_def;
 pub mod builtins_instance_def;
 pub mod diluted_pool_instance_def;
@@ -9,3 +11,25 @@ pub mod mod_instance_def;
 pub mod pedersen_instance_def;
 pub mod poseidon_instance_def;
 pub mod range_check_instance_def;
+
+#[derive(Serialize, Debug, PartialEq, Copy, Clone)]
+pub struct LowRatio {
+    pub numerator: u32,
+    pub denominator: u32,
+}
+
+impl LowRatio {
+    pub fn new(numerator: u32, denominator: u32) -> Self {
+        Self {
+            numerator,
+            denominator,
+        }
+    }
+
+    pub fn new_int(numerator: u32) -> Self {
+        Self {
+            numerator,
+            denominator: 1,
+        }
+    }
+}

vm/src/types/instance_definitions/mod_instance_def.rs

@@ -1,22 +1,43 @@
 use serde::Serialize;
 
+use super::LowRatio;
+
 pub(crate) const N_WORDS: usize = 4;
 
 pub(crate) const CELLS_PER_MOD: u32 = 7;
 
 #[derive(Serialize, Debug, PartialEq, Clone)]
 pub(crate) struct ModInstanceDef {
-    pub(crate) ratio: Option<u32>,
+    pub(crate) ratio: Option<LowRatio>,
     pub(crate) word_bit_len: u32,
     pub(crate) batch_size: usize,
 }
 
 impl ModInstanceDef {
     pub(crate) fn new(ratio: Option<u32>, batch_size: usize, word_bit_len: u32) -> Self {
         ModInstanceDef {
-            ratio,
+            ratio: ratio.map(LowRatio::new_int),
             word_bit_len,
             batch_size,
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[cfg(target_arch = "wasm32")]
+    use wasm_bindgen_test::*;
+
+    #[test]
+    #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)]
+    fn test_new() {
+        let builtin_instance = ModInstanceDef {
+            ratio: Some(LowRatio::new_int(10)),
+            word_bit_len: 3,
+            batch_size: 3,
+        };
+        assert_eq!(ModInstanceDef::new(Some(10), 3, 3), builtin_instance);
+    }
+}