-
Notifications
You must be signed in to change notification settings - Fork 77
[ON HOLD] Speed up startup by precalculating hashes and precompiling contracts #864
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 10 commits
fcdbe1c
7e5f029
16b8ef0
12e3db1
d206d03
0b9635d
795905b
bd17aad
bc4fda1
129b178
a3253df
d890392
8f302ce
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. How does removing debug data affect behavior on error and the available stack trace? |
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,11 +10,20 @@ license-file.workspace = true | |
|
|
||
| [features] | ||
| testing = [] | ||
| fastpath = [] | ||
|
|
||
| [lints] | ||
| workspace = true | ||
|
|
||
| [build-dependencies] | ||
| serde = { workspace = true } | ||
| serde_json = { workspace = true } | ||
| blake3 = "1.5" | ||
| usc = { workspace = true } | ||
| serde_json_canonicalizer = "0.3.1" | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Shouldn't this be in the root Cargo.toml? Applies to L26 and L61. |
||
|
|
||
| [dependencies] | ||
| blake3 = "1.5" | ||
| base64 = { workspace = true } | ||
| blockifier = { workspace = true } | ||
| cairo-lang-starknet-classes = { workspace = true } | ||
|
|
@@ -49,6 +58,7 @@ cairo-lang-sierra-generator = { workspace = true } | |
| cairo-lang-sierra-to-casm = { workspace = true } | ||
| cairo-lang-syntax = { workspace = true } | ||
| cairo-lang-utils = { workspace = true } | ||
| serde_json_canonicalizer = "0.3.1" | ||
|
|
||
| [dev-dependencies] | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| // build.rs | ||
| use std::path::PathBuf; | ||
| use std::{env, fs}; | ||
|
|
||
| use serde_json::Value; | ||
| use serde_json_canonicalizer::to_vec as canonical_to_vec; | ||
|
|
||
| fn compile_at_build_time(input: &Value) -> serde_json::Value { | ||
| usc::compile_contract(input.clone()) | ||
| .unwrap_or_else(|e| panic!("usc::compile_contract failed in build.rs: {e}")) | ||
| } | ||
|
|
||
| #[allow(clippy::expect_used)] | ||
| fn main() { | ||
| let manifest_dir = | ||
| PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("Failed to get manifest directory")); | ||
| let input_dir = manifest_dir.join("precompiled").join("inputs"); | ||
|
|
||
| // let fastpath_enabled = env::var("CARGO_FEATURE_FASTPATH").is_ok(); | ||
|
|
||
| let out_dir = PathBuf::from(env::var("OUT_DIR").expect("Failed to get output directory")); | ||
| let gen_path = out_dir.join("usc_fastpath.rs"); | ||
|
|
||
| // if !fastpath_enabled { | ||
| // // feature disabled → emit a stub | ||
| // fs::write(&gen_path, "pub fn lookup(_: &str) -> Option<&'static [u8]> { None }\n") | ||
| // .unwrap(); | ||
| // return; | ||
| // } | ||
|
|
||
| let mut json_files = vec![]; | ||
| if input_dir.exists() { | ||
| for e in fs::read_dir(&input_dir).expect("Failed to read precompiled inputs directory") { | ||
| let p = e.expect("Failed to read directory entry").path(); | ||
| if p.extension().and_then(|s| s.to_str()) == Some("json") { | ||
| json_files.push(p); | ||
| } | ||
| } | ||
| json_files.sort(); | ||
| } | ||
|
|
||
| if json_files.is_empty() { | ||
| fs::write(&gen_path, "pub fn lookup(_: &str) -> Option<&'static [u8]> { None }\n") | ||
| .expect("Failed to write empty lookup function"); | ||
| println!("cargo:rerun-if-changed=precompiled/inputs"); | ||
| println!("cargo:rerun-if-changed=build.rs"); | ||
| return; | ||
| } | ||
|
|
||
| let mut code = String::from("// @generated — DO NOT EDIT\n"); | ||
|
|
||
| // Emit OUTPUT_* and DIGEST_* constants | ||
| for (idx, path) in json_files.iter().enumerate() { | ||
| let raw = fs::read(path).expect("Failed to read JSON file"); | ||
| let input_val: Value = serde_json::from_slice(&raw).expect("Failed to parse JSON file"); | ||
|
|
||
| let canon_bytes = canonical_to_vec(&input_val).expect("Failed to canonicalize JSON"); | ||
|
|
||
| // Use blake3 for hashing instead of sha256 and convert to hex without using hex crate | ||
| let hash = blake3::hash(&canon_bytes).to_string(); | ||
|
|
||
| let casm_json = compile_at_build_time(&input_val); | ||
| let casm_bytes = | ||
| serde_json::to_vec(&casm_json).expect("Failed to serialize compiled contract to JSON"); | ||
|
|
||
| code.push_str(&format!("pub static OUTPUT_{idx}: &[u8] = &{:?};\n", casm_bytes)); | ||
| code.push_str(&format!("pub const DIGEST_{idx}: &str = \"{hash}\";\n")); | ||
| } | ||
|
|
||
| // Emit lookup() | ||
| code.push_str( | ||
| "\npub fn lookup(hash_hex: &str) -> Option<&'static [u8]> {\n match hash_hex {\n", | ||
| ); | ||
| for idx in 0..json_files.len() { | ||
| code.push_str(&format!(" DIGEST_{idx} => Some(OUTPUT_{idx}),\n")); | ||
| } | ||
| code.push_str(" _ => None,\n }\n}\n"); | ||
|
|
||
| fs::write(&gen_path, code).expect("Failed to write generated code to output file"); | ||
|
|
||
| println!("cargo:rerun-if-changed=precompiled/inputs"); | ||
| println!("cargo:rerun-if-changed=build.rs"); | ||
| } |
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,6 +6,11 @@ use serde_json::ser::Formatter; | |
| use serde_json::{Map, Value}; | ||
|
|
||
| use crate::error::{DevnetResult, Error, JsonError}; | ||
| #[cfg(not(clippy))] | ||
| mod usc_fastpath { | ||
| #![allow(dead_code)] | ||
| include!(concat!(env!("OUT_DIR"), "/usc_fastpath.rs")); | ||
| } | ||
|
|
||
| /// The preserve_order feature enabled in the serde_json crate | ||
| /// removing a key from the object changes the order of the keys | ||
|
|
@@ -87,6 +92,16 @@ impl Formatter for StarknetFormatter { | |
| } | ||
| } | ||
|
|
||
| pub fn canonical_serde_hash(v: &serde_json::Value) -> Result<String, serde_json::Error> { | ||
| let bytes = match serde_json_canonicalizer::to_vec(v) { | ||
| Ok(bytes) => bytes, | ||
| Err(_) => serde_json::to_vec(v)?, | ||
| }; | ||
|
|
||
| let hash = blake3::hash(&bytes).to_string(); | ||
| Ok(hash) | ||
| } | ||
|
|
||
| pub fn compile_sierra_contract(sierra_contract: &ContractClass) -> DevnetResult<CasmContractClass> { | ||
| let sierra_contract_json = serde_json::to_value(sierra_contract) | ||
| .map_err(|err| Error::JsonError(JsonError::SerdeJsonError(err)))?; | ||
|
|
@@ -95,8 +110,16 @@ pub fn compile_sierra_contract(sierra_contract: &ContractClass) -> DevnetResult< | |
| } | ||
|
|
||
| pub fn compile_sierra_contract_json( | ||
| sierra_contract_json: serde_json::Value, | ||
| sierra_contract_json: Value, | ||
| ) -> DevnetResult<CasmContractClass> { | ||
| #[cfg(not(clippy))] | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why was this line necessary? |
||
| if let Ok(Some(bytes)) = | ||
| canonical_serde_hash(&sierra_contract_json).map(|h| usc_fastpath::lookup(&h)) | ||
| { | ||
| return serde_json::from_slice::<CasmContractClass>(bytes) | ||
| .map_err(|err| Error::JsonError(JsonError::SerdeJsonError(err))); | ||
| } | ||
|
|
||
| let casm_json = usc::compile_contract(sierra_contract_json) | ||
| .map_err(|err| Error::SierraCompilationError { reason: err.to_string() })?; | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This can be minified