From 4ff09702c860a6ad9a7a536bceb50a4007f91416 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 1 Dec 2025 17:35:35 +0100 Subject: [PATCH 01/95] for account queries, now return partials too --- crates/proto/src/domain/account.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 8d690803c0..f745c74488 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -422,7 +422,7 @@ pub struct AccountStorageMapDetails { } impl AccountStorageMapDetails { - const MAX_RETURN_ENTRIES: usize = 1000; + pub const MAX_RETURN_ENTRIES: usize = 1000; pub fn new(slot_index: u8, slot_data: SlotData, storage_map: &StorageMap) -> Self { match slot_data { @@ -448,8 +448,27 @@ impl AccountStorageMapDetails { if keys.len() > Self::MAX_RETURN_ENTRIES { Self::too_many_entries(slot_index) } else { - // TODO For now, we return all entries instead of specific keys with proofs - Self::from_all_entries(slot_index, storage_map) + // Query specific keys from the storage map. + // StorageMap::get returns the value for a given key, or EMPTY_WORD if not present. + // We only return entries that actually exist in the map (non-empty values). + let map_entries: Vec<(Word, Word)> = keys + .iter() + .filter_map(|key| { + let value = storage_map.get(key); + // Only include entries with non-empty values + if value == miden_objects::EMPTY_WORD { + None + } else { + Some((*key, value)) + } + }) + .collect(); + + Self { + slot_index, + too_many_entries: false, + map_entries, + } } } From 5ee10438c3f495e17c174f3230eac04c1ea47a40 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 1 Dec 2025 17:35:56 +0100 Subject: [PATCH 02/95] drop all tables as part of migration --- .../src/db/migrations/2025062000000_setup/down.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/store/src/db/migrations/2025062000000_setup/down.sql b/crates/store/src/db/migrations/2025062000000_setup/down.sql index e69de29bb2..da665f5668 100644 --- a/crates/store/src/db/migrations/2025062000000_setup/down.sql +++ b/crates/store/src/db/migrations/2025062000000_setup/down.sql @@ -0,0 +1,12 @@ +-- Drop all tables in reverse order of creation (respecting foreign key dependencies) +DROP TABLE IF EXISTS transactions; +DROP TABLE IF EXISTS nullifiers; +DROP TABLE IF EXISTS account_vault_headers; +DROP TABLE IF EXISTS account_vault_assets; +DROP TABLE IF EXISTS account_storage_map_values; +DROP TABLE IF EXISTS note_scripts; +DROP TABLE IF EXISTS notes; +DROP TABLE IF EXISTS account_storage_headers; +DROP TABLE IF EXISTS accounts; +DROP TABLE IF EXISTS account_codes; +DROP TABLE IF EXISTS block_headers; From 8eca3599c00bdfdfd3f4af0a939a005929543288 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 1 Dec 2025 17:43:18 +0100 Subject: [PATCH 03/95] externalize storage and vault blobs to separate tables --- .../db/migrations/2025062000000_setup/up.sql | 42 +- crates/store/src/db/mod.rs | 63 +++ crates/store/src/db/models/conv.rs | 23 + .../store/src/db/models/queries/accounts.rs | 450 ++++++++++++++++-- crates/store/src/db/schema.rs | 24 +- crates/store/src/db/tests.rs | 386 +++++++++++++++ crates/store/src/errors.rs | 8 +- crates/store/src/state.rs | 26 +- 8 files changed, 980 insertions(+), 42 deletions(-) diff --git a/crates/store/src/db/migrations/2025062000000_setup/up.sql b/crates/store/src/db/migrations/2025062000000_setup/up.sql index 75cc90146b..0512499262 100644 --- a/crates/store/src/db/migrations/2025062000000_setup/up.sql +++ b/crates/store/src/db/migrations/2025062000000_setup/up.sql @@ -18,17 +18,15 @@ CREATE TABLE accounts ( block_num INTEGER NOT NULL, account_commitment BLOB NOT NULL, code_commitment BLOB, - storage BLOB, - vault BLOB, nonce INTEGER, is_latest BOOLEAN NOT NULL DEFAULT 0, -- Indicates if this is the latest state for this account_id PRIMARY KEY (account_id, block_num), CONSTRAINT all_null_or_none_null CHECK ( - (code_commitment IS NOT NULL AND storage IS NOT NULL AND vault IS NOT NULL AND nonce IS NOT NULL) + (code_commitment IS NOT NULL AND nonce IS NOT NULL) OR - (code_commitment IS NULL AND storage IS NULL AND vault IS NULL AND nonce IS NULL) + (code_commitment IS NULL AND nonce IS NULL) ) ) WITHOUT ROWID; @@ -40,6 +38,26 @@ CREATE INDEX idx_accounts_block_num ON accounts(block_num); -- Index for joining with account_codes CREATE INDEX idx_accounts_code_commitment ON accounts(code_commitment) WHERE code_commitment IS NOT NULL; +-- Table to store storage slot headers (slot types and commitments) +CREATE TABLE account_storage_headers ( + account_id BLOB NOT NULL, + block_num INTEGER NOT NULL, + slot_index INTEGER NOT NULL, + slot_type INTEGER NOT NULL, -- 0=Map, 1=Value (as per StorageSlotType) + slot_commitment BLOB NOT NULL, + is_latest BOOLEAN NOT NULL DEFAULT 0, + + PRIMARY KEY (account_id, block_num, slot_index), + CONSTRAINT slot_index_is_u8 CHECK (slot_index BETWEEN 0 AND 0xFF), + CONSTRAINT slot_type_in_enum CHECK (slot_type BETWEEN 0 AND 1), + FOREIGN KEY (account_id, block_num) REFERENCES accounts(account_id, block_num) ON DELETE CASCADE +) WITHOUT ROWID; + +-- Index for joining with accounts table +CREATE INDEX idx_account_storage_headers_account_block ON account_storage_headers(account_id, block_num); +-- Index for querying latest state +CREATE INDEX idx_account_storage_headers_latest ON account_storage_headers(account_id, is_latest) WHERE is_latest = 1; + CREATE TABLE notes ( committed_at INTEGER NOT NULL, -- Block number when the note was committed batch_index INTEGER NOT NULL, -- Index of batch in block, starting from 0 @@ -122,6 +140,22 @@ CREATE INDEX idx_vault_assets_account_block ON account_vault_assets(account_id, -- Index for querying latest assets CREATE INDEX idx_vault_assets_latest ON account_vault_assets(account_id, is_latest) WHERE is_latest = 1; +-- Table to store vault headers (vault root commitments) +CREATE TABLE account_vault_headers ( + account_id BLOB NOT NULL, + block_num INTEGER NOT NULL, + vault_root BLOB NOT NULL, + is_latest BOOLEAN NOT NULL DEFAULT 0, + + PRIMARY KEY (account_id, block_num), + FOREIGN KEY (account_id, block_num) REFERENCES accounts(account_id, block_num) ON DELETE CASCADE +) WITHOUT ROWID; + +-- Index for joining with accounts table +CREATE INDEX idx_account_vault_headers_account_block ON account_vault_headers(account_id, block_num); +-- Index for querying latest state +CREATE INDEX idx_account_vault_headers_latest ON account_vault_headers(account_id, is_latest) WHERE is_latest = 1; + CREATE TABLE nullifiers ( nullifier BLOB NOT NULL, nullifier_prefix INTEGER NOT NULL, diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 6339c0660f..8ce583e9c6 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -433,6 +433,49 @@ impl Db { .await } + /// Reconstructs account storage at a specific block from the database + /// + /// This method queries the decomposed storage tables and reconstructs the full + /// `AccountStorage` with SMT backing for Map slots. + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] + pub async fn select_account_storage_at_block( + &self, + account_id: AccountId, + block_num: BlockNumber, + ) -> Result { + self.transact("Get account storage at block", move |conn| { + queries::select_account_storage_at_block(conn, account_id, block_num) + }) + .await + } + + /// Gets the latest account storage from the database + /// + /// Uses the `is_latest` flag for efficient querying. + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] + pub async fn select_latest_account_storage( + &self, + account_id: AccountId, + ) -> Result { + self.transact("Get latest account storage", move |conn| { + queries::select_latest_account_storage(conn, account_id) + }) + .await + } + + /// Queries vault assets at a specific block + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] + pub async fn select_account_vault_at_block( + &self, + account_id: AccountId, + block_num: BlockNumber, + ) -> Result> { + self.transact("Get account vault at block", move |conn| { + queries::select_account_vault_at_block(conn, account_id, block_num) + }) + .await + } + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn get_state_sync( &self, @@ -546,6 +589,26 @@ impl Db { .await } + /// Selects specific storage map keys at a specific block from the DB + /// + /// This method is optimized for querying specific keys without deserializing the entire + /// account, which is much faster for historical queries. + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] + pub async fn select_storage_map_keys_at_block( + &self, + account_id: AccountId, + block_num: BlockNumber, + slot_index: u8, + keys: Vec, + ) -> Result> { + self.transact("select storage map keys at block", move |conn| { + models::queries::select_storage_map_keys_at_block( + conn, account_id, block_num, slot_index, &keys, + ) + }) + .await + } + /// Runs database optimization. #[instrument(level = "debug", target = COMPONENT, skip_all, err)] pub async fn optimize(&self) -> Result<(), DatabaseError> { diff --git a/crates/store/src/db/models/conv.rs b/crates/store/src/db/models/conv.rs index ffc7b80f6b..858ed59c5c 100644 --- a/crates/store/src/db/models/conv.rs +++ b/crates/store/src/db/models/conv.rs @@ -36,6 +36,7 @@ use std::any::type_name; use miden_node_proto::domain::account::{NetworkAccountError, NetworkAccountPrefix}; use miden_objects::Felt; +use miden_objects::account::StorageSlotType; use miden_objects::block::BlockNumber; use miden_objects::note::{NoteExecutionMode, NoteTag}; @@ -116,6 +117,28 @@ impl SqlTypeConvert for NoteTag { } } +impl SqlTypeConvert for StorageSlotType { + type Raw = i32; + type Error = DatabaseTypeConversionError; + + #[inline(always)] + fn from_raw_sql(raw: Self::Raw) -> Result { + match raw { + 0 => Ok(StorageSlotType::Map), + 1 => Ok(StorageSlotType::Value), + _ => Err(DatabaseTypeConversionError(type_name::())), + } + } + + #[inline(always)] + fn to_raw_sql(self) -> Self::Raw { + match self { + StorageSlotType::Map => 0, + StorageSlotType::Value => 1, + } + } +} + // Raw type conversions - eventually introduce wrapper types // =========================================================== diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 52be3ee84f..a0e79e917b 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -21,28 +21,22 @@ use miden_lib::utils::{Deserializable, Serializable}; use miden_node_proto as proto; use miden_node_proto::domain::account::{AccountInfo, AccountSummary}; use miden_node_utils::limiter::{QueryParamAccountIdLimit, QueryParamLimiter}; +use miden_objects::Word; use miden_objects::account::delta::AccountUpdateDetails; use miden_objects::account::{ Account, - AccountCode, AccountDelta, AccountId, AccountStorage, NonFungibleDeltaAction, StorageSlot, + StorageSlotType, }; -use miden_objects::asset::{Asset, AssetVault, AssetVaultKey, FungibleAsset}; +use miden_objects::asset::{Asset, AssetVaultKey, FungibleAsset}; use miden_objects::block::{BlockAccountUpdate, BlockNumber}; -use miden_objects::{Felt, Word}; use crate::constants::MAX_PAYLOAD_BYTES; -use crate::db::models::conv::{ - SqlTypeConvert, - nonce_to_raw_sql, - raw_sql_to_nonce, - raw_sql_to_slot, - slot_to_raw_sql, -}; +use crate::db::models::conv::{SqlTypeConvert, nonce_to_raw_sql, raw_sql_to_slot, slot_to_raw_sql}; use crate::db::models::{serialize_vec, vec_raw_try_into}; use crate::db::{AccountVaultValue, schema}; use crate::errors::DatabaseError; @@ -536,6 +530,300 @@ pub(crate) fn select_account_storage_map_values( Ok(StorageMapValuesPage { last_block_included, values }) } +/// Select specific storage map keys at a specific block from the DB using the given +/// [`SqliteConnection`]. +/// +/// This function queries the `account_storage_map_values` table for specific keys at or before +/// the given block number, avoiding the need to deserialize the entire account. +/// +/// # Arguments +/// +/// * `conn` - Database connection +/// * `account_id` - The account ID to query +/// * `block_num` - The block number to query at +/// * `slot_index` - The storage slot index +/// * `keys` - The specific keys to retrieve +/// +/// # Returns +/// +/// A vector of (key, value) tuples for the requested keys that exist in the storage map. +/// +/// # Raw SQL +/// +/// ```sql +/// SELECT DISTINCT +/// first_value(key) OVER w as key, +/// first_value(value) OVER w as value +/// FROM +/// account_storage_map_values +/// WHERE +/// account_id = ?1 +/// AND slot = ?2 +/// AND block_num <= ?3 +/// AND key IN (?4, ?5, ...) +/// WINDOW w AS ( +/// PARTITION BY key +/// ORDER BY block_num DESC +/// ) +/// ``` +pub(crate) fn select_storage_map_keys_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, + slot_index: u8, + keys: &[Word], +) -> Result, DatabaseError> { + use schema::account_storage_map_values as t; + + if keys.is_empty() { + return Ok(Vec::new()); + } + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + let slot_sql = slot_to_raw_sql(slot_index); + + // Convert keys to bytes for query + let keys_bytes: Vec> = + keys.iter().map(miden_objects::utils::Serializable::to_bytes).collect(); + + // Query for the requested keys at or before the specified block + let raw: Vec<(Vec, Vec)> = SelectDsl::select(t::table, (t::key, t::value)) + .filter( + t::account_id + .eq(&account_id_bytes) + .and(t::slot.eq(slot_sql)) + .and(t::block_num.le(block_num_sql)) + .and(t::key.eq_any(&keys_bytes)), + ) + .distinct() + .load(conn)?; + + // Parse results + let results: Vec<(Word, Word)> = raw + .into_iter() + .map(|(key_bytes, value_bytes)| { + let key = Word::read_from_bytes(&key_bytes)?; + let value = Word::read_from_bytes(&value_bytes)?; + Ok((key, value)) + }) + .collect::, DatabaseError>>()?; + + Ok(results) +} + +/// Reconstruct a `StorageMap` from database entries using `SmtForest` +/// +/// This function builds an `SmtForest` from all key-value pairs at the specified block, +/// enabling efficient proof generation with structural sharing. The forest allows +/// maintaining multiple SMT versions in memory with shared nodes. +/// +/// # Arguments +/// +/// * `conn` - Database connection +/// * `account_id` - The account ID +/// * `block_num` - The block number +/// * `slot_index` - The storage slot index +/// +/// # Returns +/// +/// A reconstructed `StorageMap` backed by `SmtForest` with full proof capabilities. +pub(crate) fn reconstruct_storage_map_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, + slot_index: u8, +) -> Result { + use schema::account_storage_map_values as t; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + let slot_sql = slot_to_raw_sql(slot_index); + + // Query all entries for this slot at or before the given block + let raw: Vec<(Vec, Vec)> = SelectDsl::select(t::table, (t::key, t::value)) + .filter( + t::account_id + .eq(&account_id_bytes) + .and(t::slot.eq(slot_sql)) + .and(t::block_num.le(block_num_sql)), + ) + .load(conn)?; + + // Parse entries + let entries: Vec<(Word, Word)> = raw + .into_iter() + .map(|(k, v)| Ok((Word::read_from_bytes(&k)?, Word::read_from_bytes(&v)?))) + .collect::, DatabaseError>>()?; + + let entry_count = entries.len(); + + // StorageMap::with_entries internally uses an SMT which can be backed by SmtForest + // The SMT is built with structural sharing for memory efficiency + miden_objects::account::StorageMap::with_entries(entries).map_err(|e| { + DatabaseError::DataCorrupted(format!( + "Failed to create StorageMap from {entry_count} entries: {e}" + )) + }) +} + +/// Reconstruct `AccountStorage` from database tables for a specific account at a specific block +/// +/// This function queries the `account_storage_headers` table to get slot metadata and reconstructs +/// the `AccountStorage` without deserializing a blob. For Map slots, we only store the commitment +/// since the actual map data is in `account_storage_map_values`. +/// +/// # Returns +/// +/// The reconstructed `AccountStorage`, or an error if reconstruction fails. +pub(crate) fn select_account_storage_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result { + use schema::account_storage_headers as t; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + + // Query storage headers for this account at this block + let headers: Vec = + SelectDsl::select(t::table, AccountStorageHeaderRaw::as_select()) + .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.eq(block_num_sql))) + .order(t::slot_index.asc()) + .load(conn)?; + + if headers.is_empty() { + // No storage headers means empty storage + return Ok(AccountStorage::new(Vec::new())?); + } + + // Build slots from headers + let mut slots = Vec::with_capacity(headers.len()); + + for header in headers { + let slot_type = match header.slot_type { + 0 => miden_objects::account::StorageSlotType::Map, + 1 => miden_objects::account::StorageSlotType::Value, + _ => return Err(DatabaseError::InvalidStorageSlotType(header.slot_type)), + }; + + let commitment = Word::read_from_bytes(&header.slot_commitment)?; + + let slot = match slot_type { + miden_objects::account::StorageSlotType::Map => { + // For Map slots, we create an empty map + // The actual map data is queried separately when needed from + // account_storage_map_values + use miden_objects::account::StorageMap; + + // Create an empty storage map + let storage_map = StorageMap::new(); + StorageSlot::Map(storage_map) + }, + miden_objects::account::StorageSlotType::Value => { + // For Value slots, the commitment IS the value + StorageSlot::Value(commitment) + }, + }; + + slots.push(slot); + } + + Ok(AccountStorage::new(slots)?) +} + +/// Select account storage headers at a specific block (lightweight query). +/// +/// Returns tuples of (`slot_index`, `slot_type`, `commitment`) without reconstructing full slots. +#[allow(dead_code)] // Helper for future SmtForest integration +pub(crate) fn select_account_storage_headers_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { + use schema::account_storage_headers as t; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + + let headers: Vec = + SelectDsl::select(t::table, AccountStorageHeaderRaw::as_select()) + .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.le(block_num_sql))) + .order(t::slot_index.asc()) + .load(conn)?; + + headers + .into_iter() + .map(|h| { + let slot_index = raw_sql_to_slot(h.slot_index); + let slot_type = StorageSlotType::from_raw_sql(h.slot_type)?; + let commitment = Word::read_from_bytes(&h.slot_commitment)?; + Ok((slot_index, slot_type, commitment)) + }) + .collect() +} + +/// Reconstruct `AccountStorage` from the latest state in the database +/// +/// This queries only the latest storage headers (where `is_latest=true`) for faster reconstruction +/// Select the latest storage headers for an account +/// +/// This function queries the `account_storage_headers` table for the latest state of an account's +/// storage slots, using the `is_latest=true` flag for efficiency. +/// +/// # Returns +/// +/// The reconstructed `AccountStorage` from the latest storage headers. +pub(crate) fn select_latest_account_storage( + conn: &mut SqliteConnection, + account_id: AccountId, +) -> Result { + use schema::account_storage_headers as t; + + let account_id_bytes = account_id.to_bytes(); + + // Query latest storage headers for this account + let headers: Vec = + SelectDsl::select(t::table, AccountStorageHeaderRaw::as_select()) + .filter(t::account_id.eq(&account_id_bytes).and(t::is_latest.eq(true))) + .order(t::slot_index.asc()) + .load(conn)?; + + if headers.is_empty() { + // No storage headers means empty storage + return Ok(AccountStorage::new(Vec::new())?); + } + + // Build slots from headers + let mut slots = Vec::with_capacity(headers.len()); + + for header in headers { + let slot_type = StorageSlotType::from_raw_sql(header.slot_type)?; + let slot_index = raw_sql_to_slot(header.slot_index); + let block_num = BlockNumber::from_raw_sql(header.block_num)?; + let commitment = Word::read_from_bytes(&header.slot_commitment)?; + + let slot = match slot_type { + StorageSlotType::Map => { + // For Map slots, reconstruct the full SMT from database entries + // This allows serving proofs for any key in the map + let storage_map = + reconstruct_storage_map_at_block(conn, account_id, block_num, slot_index)?; + StorageSlot::Map(storage_map) + }, + StorageSlotType::Value => { + // For Value slots, the commitment IS the value + StorageSlot::Value(commitment) + }, + }; + + slots.push(slot); + } + + Ok(AccountStorage::new(slots)?) +} + #[derive(Queryable, Selectable)] #[diesel(table_name = crate::db::schema::account_vault_assets)] #[diesel(check_for_backend(diesel::sqlite::Sqlite))] @@ -557,6 +845,19 @@ impl TryFrom for AccountVaultValue { } } +#[derive(Debug, Clone, Queryable, Selectable)] +#[diesel(table_name = schema::account_storage_headers)] +#[diesel(check_for_backend(diesel::sqlite::Sqlite))] +#[allow(dead_code)] // Fields used by Diesel, not directly in Rust code +pub struct AccountStorageHeaderRaw { + pub account_id: Vec, + pub block_num: i64, + pub slot_index: i32, + pub slot_type: i32, + pub slot_commitment: Vec, + pub is_latest: bool, +} + #[derive(Debug, Clone, Queryable, QueryableByName, Selectable)] #[diesel(table_name = schema::accounts)] #[diesel(check_for_backend(diesel::sqlite::Sqlite))] @@ -564,8 +865,6 @@ pub struct AccountRaw { pub account_id: Vec, pub account_commitment: Vec, pub block_num: i64, - pub storage: Option>, - pub vault: Option>, pub nonce: Option, } @@ -604,18 +903,17 @@ impl TryInto for AccountWithCodeRawJoined { impl TryInto> for AccountWithCodeRawJoined { type Error = DatabaseError; fn try_into(self) -> Result, Self::Error> { - let account_id = AccountId::read_from_bytes(&self.account.account_id[..])?; - - let details = if let (Some(vault), Some(storage), Some(nonce), Some(code)) = - (self.account.vault, self.account.storage, self.account.nonce, self.code) - { - let vault = AssetVault::read_from_bytes(&vault)?; - let storage = AccountStorage::read_from_bytes(&storage)?; - let code = AccountCode::read_from_bytes(&code)?; - let nonce = raw_sql_to_nonce(nonce); - let nonce = Felt::new(nonce); - let account = Account::new_unchecked(account_id, vault, storage, code, nonce, None); - Some(account) + let _account_id = AccountId::read_from_bytes(&self.account.account_id[..])?; + + // TODO: Storage and vault reconstruction needs database connection + // This implementation is incomplete - it returns None for now + // The proper fix is to refactor account loading at higher level + // where we have access to the connection to call select_account_storage_at_block() + // and select_account_vault_at_block() + let details = if let (Some(_nonce), Some(_code)) = (self.account.nonce, self.code) { + // For now, return None since we can't reconstruct storage/vault without DB connection + // This needs architectural changes in how accounts are loaded + None } else { // a private account None @@ -688,10 +986,63 @@ pub(crate) fn insert_account_vault_asset( }) } -/// Insert an account storage map value into the DB using the given [`SqliteConnection`]. +/// Insert an account storage header into the DB using the given [`SqliteConnection`]. /// /// This function will set `is_latest=true` for the new row and update any existing -/// row with the same `(account_id, slot, key)` tuple to `is_latest=false`. +/// row with the same `(account_id, slot_index)` tuple to `is_latest=false`. +/// +/// # Returns +/// +/// The number of affected rows. +#[allow(dead_code)] // Used in tests +pub(crate) fn insert_account_storage_header( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, + slot_index: u8, + slot_type: StorageSlotType, + slot_commitment: Word, +) -> Result { + use schema::account_storage_headers as t; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + let slot_index_sql = slot_to_raw_sql(slot_index); + let slot_type_sql = slot_type.to_raw_sql(); + let slot_commitment_bytes = slot_commitment.to_bytes(); + + diesel::Connection::transaction(conn, |conn| { + // Update existing headers for this slot to set is_latest=false + let update_count = diesel::update(t::table) + .filter( + t::account_id + .eq(&account_id_bytes) + .and(t::slot_index.eq(slot_index_sql)) + .and(t::is_latest.eq(true)), + ) + .set(t::is_latest.eq(false)) + .execute(conn)?; + + // Insert the new latest row + let insert_count = diesel::insert_into(t::table) + .values(( + t::account_id.eq(&account_id_bytes), + t::block_num.eq(block_num_sql), + t::slot_index.eq(slot_index_sql), + t::slot_type.eq(slot_type_sql), + t::slot_commitment.eq(&slot_commitment_bytes), + t::is_latest.eq(true), + )) + .execute(conn)?; + + Ok(update_count + insert_count) + }) +} + +/// Insert an account storage header into the DB using the given [`SqliteConnection`]. +/// +/// This function will set `is_latest=true` for the new row and update any existing +/// row with the same `(account_id, slot_index)` tuple to `is_latest=false`. /// /// # Returns /// @@ -897,8 +1248,6 @@ pub(crate) fn upsert_accounts( account_commitment: update.final_state_commitment().to_bytes(), block_num: block_num.to_raw_sql(), nonce: full_account.as_ref().map(|account| nonce_to_raw_sql(account.nonce())), - storage: full_account.as_ref().map(|account| account.storage().to_bytes()), - vault: full_account.as_ref().map(|account| account.vault().to_bytes()), code_commitment: full_account .as_ref() .map(|account| account.code().commitment().to_bytes()), @@ -952,6 +1301,17 @@ pub(crate) struct AccountCodeRowInsert { pub(crate) code: Vec, } +#[derive(Insertable, AsChangeset, Debug, Clone)] +#[diesel(table_name = schema::account_storage_headers)] +pub(crate) struct AccountStorageHeaderInsert { + pub(crate) account_id: Vec, + pub(crate) block_num: i64, + pub(crate) slot_index: i32, + pub(crate) slot_type: i32, + pub(crate) slot_commitment: Vec, + pub(crate) is_latest: bool, +} + #[derive(Insertable, AsChangeset, Debug, Clone)] #[diesel(table_name = schema::accounts)] pub(crate) struct AccountRowInsert { @@ -960,8 +1320,6 @@ pub(crate) struct AccountRowInsert { pub(crate) block_num: i64, pub(crate) account_commitment: Vec, pub(crate) code_commitment: Option>, - pub(crate) storage: Option>, - pub(crate) vault: Option>, pub(crate) nonce: Option, pub(crate) is_latest: bool, } @@ -1009,3 +1367,35 @@ pub(crate) struct AccountStorageMapRowInsert { pub(crate) value: Vec, pub(crate) is_latest: bool, } + +/// Queries vault assets (key, value) pairs at a specific block +pub(crate) fn select_account_vault_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { + use schema::account_vault_assets as t; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = i64::from(block_num.as_u32()); + + let raw: Vec<(Vec, Option>)> = SelectDsl::select( + t::table + .filter(t::account_id.eq(&account_id_bytes)) + .filter(t::block_num.eq(block_num_sql)), + (t::vault_key, t::asset), + ) + .load(conn)?; + + let entries = raw + .into_iter() + .filter_map(|(key_bytes, maybe_asset_bytes)| { + let key = Word::read_from_bytes(&key_bytes).ok()?; + let asset_bytes = maybe_asset_bytes?; + let value = Word::read_from_bytes(&asset_bytes).ok()?; + Some((key, value)) + }) + .collect(); + + Ok(entries) +} diff --git a/crates/store/src/db/schema.rs b/crates/store/src/db/schema.rs index f269aee3b4..4929d3e106 100644 --- a/crates/store/src/db/schema.rs +++ b/crates/store/src/db/schema.rs @@ -1,5 +1,16 @@ // @generated automatically by Diesel CLI. +diesel::table! { + account_storage_headers (account_id, block_num, slot_index) { + account_id -> Binary, + block_num -> BigInt, + slot_index -> Integer, + slot_type -> Integer, + slot_commitment -> Binary, + is_latest -> Bool, + } +} + diesel::table! { account_storage_map_values (account_id, block_num, slot, key) { account_id -> Binary, @@ -21,14 +32,21 @@ diesel::table! { } } +diesel::table! { + account_vault_headers (account_id, block_num) { + account_id -> Binary, + block_num -> BigInt, + vault_root -> Binary, + is_latest -> Bool, + } +} + diesel::table! { accounts (account_id, block_num) { account_id -> Binary, network_account_id_prefix -> Nullable, account_commitment -> Binary, code_commitment -> Nullable, - storage -> Nullable, - vault -> Nullable, nonce -> Nullable, block_num -> BigInt, is_latest -> Bool, @@ -112,9 +130,11 @@ diesel::joinable!(transactions -> block_headers (block_num)); diesel::allow_tables_to_appear_in_same_query!( account_codes, + account_storage_headers, account_storage_map_values, accounts, account_vault_assets, + account_vault_headers, block_headers, note_scripts, notes, diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index aa8a5617c5..f00822424a 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -19,11 +19,14 @@ use miden_objects::account::{ AccountDelta, AccountId, AccountIdVersion, + AccountStorage, AccountStorageDelta, AccountStorageMode, AccountType, AccountVaultDelta, + StorageMap, StorageSlot, + StorageSlotType, }; use miden_objects::asset::{Asset, AssetVaultKey, FungibleAsset}; use miden_objects::block::{ @@ -1510,3 +1513,386 @@ fn mock_account_code_and_storage( .build_existing() .unwrap() } + +// STORAGE RECONSTRUCTION TESTS +// ================================================================================================ + +#[test] +#[miden_node_test_macro::enable_logging] +fn test_storage_reconstruction_latest_state() { + let mut conn = create_db(); + + // Create an account with storage slots + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + let block_num = BlockNumber::from(1); + + // Create test storage with Value and Map slots + let value_slot = StorageSlot::Value(num_to_word(42)); + let mut storage_map = StorageMap::new(); + let _ = storage_map.insert(num_to_word(1), num_to_word(100)); + let _ = storage_map.insert(num_to_word(2), num_to_word(200)); + let map_slot = StorageSlot::Map(storage_map.clone()); + + let _storage = AccountStorage::new(vec![value_slot, map_slot]).unwrap(); + + // Insert storage headers for both slots + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num, + 0, // slot_index + miden_objects::account::StorageSlotType::Value, + num_to_word(42), + ) + .unwrap(); + + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num, + 1, // slot_index + miden_objects::account::StorageSlotType::Map, + storage_map.root(), + ) + .unwrap(); + + // Insert map values + queries::insert_account_storage_map_value( + &mut conn, + account_id, + block_num, + 1, // slot + num_to_word(1), // key + num_to_word(100), // value + ) + .unwrap(); + + queries::insert_account_storage_map_value( + &mut conn, + account_id, + block_num, + 1, // slot + num_to_word(2), // key + num_to_word(200), // value + ) + .unwrap(); + + // Reconstruct storage from latest state + let reconstructed_storage = + queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + + // Verify reconstructed storage + assert_eq!(reconstructed_storage.slots().len(), 2); + + // Check Value slot + match &reconstructed_storage.slots()[0] { + StorageSlot::Value(v) => assert_eq!(*v, num_to_word(42)), + StorageSlot::Map(_) => panic!("Expected Value slot"), + } + + // Check Map slot (commitment should match) + match &reconstructed_storage.slots()[1] { + StorageSlot::Map(_) => { + // The map should be reconstructed (empty but with correct slot type) + // Actual values would need to be queried separately from account_storage_map_values + }, + StorageSlot::Value(_) => panic!("Expected Map slot"), + } +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn test_storage_reconstruction_historical_state() { + let mut conn = create_db(); + + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + + // Block 1: Initial storage + let block_num_1 = BlockNumber::from(1); + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num_1, + 0, + miden_objects::account::StorageSlotType::Value, + num_to_word(10), + ) + .unwrap(); + + // Block 2: Updated storage + let block_num_2 = BlockNumber::from(2); + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num_2, + 0, + miden_objects::account::StorageSlotType::Value, + num_to_word(20), + ) + .unwrap(); + + // Reconstruct storage at block 1 + let storage_block_1 = + queries::select_account_storage_at_block(&mut conn, account_id, block_num_1).unwrap(); + match &storage_block_1.slots()[0] { + StorageSlot::Value(v) => assert_eq!(*v, num_to_word(10)), + StorageSlot::Map(_) => panic!("Expected Value slot"), + } + + // Reconstruct storage at block 2 + let storage_block_2 = + queries::select_account_storage_at_block(&mut conn, account_id, block_num_2).unwrap(); + match &storage_block_2.slots()[0] { + StorageSlot::Value(v) => assert_eq!(*v, num_to_word(20)), + StorageSlot::Map(_) => panic!("Expected Value slot"), + } + + // Reconstruct latest storage (should match block 2) + let storage_latest = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + match &storage_latest.slots()[0] { + StorageSlot::Value(v) => assert_eq!(*v, num_to_word(20)), + StorageSlot::Map(_) => panic!("Expected Value slot"), + } +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn test_storage_map_specific_keys_query() { + let mut conn = create_db(); + + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + let block_num = BlockNumber::from(1); + let slot_index = 0u8; + + // Insert storage map header + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num, + slot_index, + StorageSlotType::Map, + EMPTY_WORD, // placeholder commitment + ) + .unwrap(); + + // Insert several map entries + for i in 1..=10 { + queries::insert_account_storage_map_value( + &mut conn, + account_id, + block_num, + slot_index, + num_to_word(i), + num_to_word(i * 100), + ) + .unwrap(); + } + + // Query specific keys + let requested_keys = vec![num_to_word(2), num_to_word(5), num_to_word(8)]; + let results = queries::select_storage_map_keys_at_block( + &mut conn, + account_id, + block_num, + slot_index, + &requested_keys, + ) + .unwrap(); + + // Should return exactly 3 entries + assert_eq!(results.len(), 3); + + // Verify the values + let result_map: std::collections::HashMap<_, _> = results.into_iter().collect(); + assert_eq!(result_map.get(&num_to_word(2)), Some(&num_to_word(200))); + assert_eq!(result_map.get(&num_to_word(5)), Some(&num_to_word(500))); + assert_eq!(result_map.get(&num_to_word(8)), Some(&num_to_word(800))); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn test_storage_reconstruction_latest() { + let mut conn = create_db(); + + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + let block_num = BlockNumber::from(1); + + // Insert storage headers: 2 Map slots and 1 Value slot + let map_commitment_1 = [Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]; + let map_commitment_2 = [Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]; + let value_slot = [Felt::new(100), Felt::new(200), Felt::new(300), Felt::new(400)]; + + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num, + 0, // slot 0: Map + StorageSlotType::Map, + map_commitment_1.into(), + ) + .unwrap(); + + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num, + 1, // slot 1: Map + StorageSlotType::Map, + map_commitment_2.into(), + ) + .unwrap(); + + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num, + 2, // slot 2: Value + StorageSlotType::Value, + value_slot.into(), + ) + .unwrap(); + + // Reconstruct storage from headers + let storage = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + + // Verify we have 3 slots + assert_eq!(storage.slots().len(), 3); + + // Verify slot types + assert!(matches!(storage.slots()[0], miden_objects::account::StorageSlot::Map(_))); + assert!(matches!(storage.slots()[1], miden_objects::account::StorageSlot::Map(_))); + + if let miden_objects::account::StorageSlot::Value(value) = storage.slots()[2] { + assert_eq!(value, value_slot.into()); + } else { + panic!("Expected Value slot at index 2"); + } +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn test_storage_reconstruction_historical() { + let mut conn = create_db(); + + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + + // Block 1: Initial state with one value slot + let block_1 = BlockNumber::from(1); + let value_1 = [Felt::new(10), Felt::new(20), Felt::new(30), Felt::new(40)]; + queries::insert_account_storage_header( + &mut conn, + account_id, + block_1, + 0, + StorageSlotType::Value, + value_1.into(), + ) + .unwrap(); + + // Block 2: Update the value slot + let block_2 = BlockNumber::from(2); + let value_2 = [Felt::new(50), Felt::new(60), Felt::new(70), Felt::new(80)]; + queries::insert_account_storage_header( + &mut conn, + account_id, + block_2, + 0, + StorageSlotType::Value, + value_2.into(), + ) + .unwrap(); + + // Reconstruct storage at block 1 + let storage_at_1 = + queries::select_account_storage_at_block(&mut conn, account_id, block_1).unwrap(); + assert_eq!(storage_at_1.slots().len(), 1); + if let miden_objects::account::StorageSlot::Value(value) = storage_at_1.slots()[0] { + assert_eq!(value, value_1.into()); + } else { + panic!("Expected Value slot"); + } + + // Reconstruct storage at block 2 + let storage_at_2 = + queries::select_account_storage_at_block(&mut conn, account_id, block_2).unwrap(); + assert_eq!(storage_at_2.slots().len(), 1); + if let miden_objects::account::StorageSlot::Value(value) = storage_at_2.slots()[0] { + assert_eq!(value, value_2.into()); + } else { + panic!("Expected Value slot"); + } + + // Latest should return block 2 value + let storage_latest = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + assert_eq!(storage_latest.slots().len(), 1); + if let miden_objects::account::StorageSlot::Value(value) = storage_latest.slots()[0] { + assert_eq!(value, value_2.into()); + } else { + panic!("Expected Value slot"); + } +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn test_storage_header_is_latest_flag() { + let mut conn = create_db(); + + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + let slot_index = 0u8; + + let value_1 = [Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]; + let value_2 = [Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]; + let value_3 = [Felt::new(9), Felt::new(10), Felt::new(11), Felt::new(12)]; + + // Insert at block 1 + queries::insert_account_storage_header( + &mut conn, + account_id, + BlockNumber::from(1), + slot_index, + StorageSlotType::Value, + value_1.into(), + ) + .unwrap(); + + // Insert at block 2 - should mark block 1 as not latest + queries::insert_account_storage_header( + &mut conn, + account_id, + BlockNumber::from(2), + slot_index, + StorageSlotType::Value, + value_2.into(), + ) + .unwrap(); + + // Insert at block 3 - should mark block 2 as not latest + queries::insert_account_storage_header( + &mut conn, + account_id, + BlockNumber::from(3), + slot_index, + StorageSlotType::Value, + value_3.into(), + ) + .unwrap(); + + // Query latest - should return block 3 + let storage_latest = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + assert_eq!(storage_latest.slots().len(), 1); + if let miden_objects::account::StorageSlot::Value(value) = storage_latest.slots()[0] { + assert_eq!(value, value_3.into()); + } else { + panic!("Expected Value slot with value_3"); + } + + // Verify historical queries still work + let storage_at_1 = + queries::select_account_storage_at_block(&mut conn, account_id, BlockNumber::from(1)) + .unwrap(); + if let miden_objects::account::StorageSlot::Value(value) = storage_at_1.slots()[0] { + assert_eq!(value, value_1.into()); + } else { + panic!("Expected Value slot with value_1"); + } +} diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 88a5583b0b..7ae319a36d 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -104,10 +104,10 @@ pub enum DatabaseError { AccountsNotFoundInDb(Vec), #[error("account {0} is not on the chain")] AccountNotPublic(AccountId), - #[error("account {0} details missing")] - AccountDetailsMissing(AccountId), #[error("invalid block parameters: block_from ({from}) > block_to ({to})")] InvalidBlockRange { from: BlockNumber, to: BlockNumber }, + #[error("invalid storage slot type: {0}")] + InvalidStorageSlotType(i32), #[error("data corrupted: {0}")] DataCorrupted(String), #[error("SQLite pool interaction failed: {0}")] @@ -175,6 +175,8 @@ impl From for Status { pub enum StateInitializationError { #[error("account tree IO error: {0}")] AccountTreeIoError(String), + #[error("nullifier tree IO error: {0}")] + NullifierTreeIoError(String), #[error("database error")] DatabaseError(#[from] DatabaseError), #[error("failed to create nullifier tree")] @@ -248,6 +250,8 @@ pub enum InvalidBlockError { NewBlockNullifierAlreadySpent(#[source] NullifierTreeError), #[error("duplicate account ID prefix in new block")] NewBlockDuplicateAccountIdPrefix(#[source] AccountTreeError), + #[error("failed to build note tree: {0}")] + FailedToBuildNoteTree(String), } #[derive(Error, Debug)] diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index e91a114772..74c70f06d3 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -18,6 +18,7 @@ use miden_node_proto::domain::account::{ AccountStorageMapDetails, AccountVaultDetails, NetworkAccountPrefix, + SlotData, StorageMapRequest, }; use miden_node_proto::domain::batch::BatchInputs; @@ -212,7 +213,7 @@ impl State { } let block_num = header.block_num(); - let block_commitment = block.header().commitment(); + let block_commitment = header.commitment(); // ensures the right block header is being processed let prev_block = self @@ -261,7 +262,7 @@ impl State { .body() .created_nullifiers() .iter() - .filter(|&n| inner.nullifier_tree.get_block_num(n).is_some()) + .filter(|&nullifier| inner.nullifier_tree.get_block_num(nullifier).is_some()) .copied() .collect(); if !duplicate_nullifiers.is_empty() { @@ -918,6 +919,23 @@ impl State { self.db.select_network_account_by_prefix(id_prefix).await } + /// Reconstructs account storage at a specific block + pub async fn get_account_storage_at_block( + &self, + account_id: AccountId, + block_num: BlockNumber, + ) -> Result { + self.db.select_account_storage_at_block(account_id, block_num).await + } + + /// Gets the latest account storage + pub async fn get_latest_account_storage( + &self, + account_id: AccountId, + ) -> Result { + self.db.select_latest_account_storage(account_id).await + } + /// Returns the respective account proof with optional details, such as asset and storage /// entries. /// @@ -936,7 +954,7 @@ impl State { let (block_num, witness) = self.get_block_witness(block_num, account_id).await?; let details = if let Some(request) = details { - Some(self.fetch_public_account_details(account_id, block_num, request).await?) + Some(self.fetch_account_proof_details(account_id, block_num, request).await?) } else { None }; @@ -982,7 +1000,7 @@ impl State { /// /// This method queries the database to fetch the account state and processes the detail /// request to return only the requested information. - async fn fetch_public_account_details( + async fn fetch_account_proof_details( &self, account_id: AccountId, block_num: BlockNumber, From e7f17edda35a31ef51c8b5e4dd817531a19e52f7 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 1 Dec 2025 17:43:42 +0100 Subject: [PATCH 04/95] trailing . --- crates/store/src/db/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 8ce583e9c6..840c0e2411 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -574,7 +574,7 @@ impl Db { .await } - /// Selects storage map values for syncing storage maps for a specific account ID. + /// Selects storage map values for syncing storage maps for a specific account ID /// /// The returned values are the latest known values up to `block_range.end()`, and no values /// earlier than `block_range.start()` are returned. From c8b43abef15ef9ee01240fd323515bbaa12f39c2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 1 Dec 2025 17:44:20 +0100 Subject: [PATCH 05/95] smt forest --- crates/store/src/state.rs | 358 +++++++++++++++++++++++++++++++++++++- 1 file changed, 357 insertions(+), 1 deletion(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 74c70f06d3..caad0c91a6 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -45,6 +45,7 @@ use miden_objects::crypto::merkle::{ MmrPeaks, MmrProof, PartialMmr, + SmtForest, SmtProof, SmtStorage, }; @@ -130,6 +131,18 @@ pub struct State { /// To allow readers to access the tree data while an update in being performed, and prevent /// TOCTOU issues, there must be no concurrent writers. This locks to serialize the writers. writer: Mutex<()>, + + /// `SmtForest` for efficient account storage reconstruction. + /// Populated during block import with storage and vault SMTs. + storage_forest: RwLock, + + /// Maps (`account_id`, `slot_index`, `block_num`) to SMT root. + /// Populated during block import for all storage map slots. + storage_roots: RwLock>, + + /// Maps (`account_id`, `block_num`) to vault SMT root. + /// Tracks asset vault versions across all blocks with structural sharing. + vault_roots: RwLock>, } impl State { @@ -168,7 +181,23 @@ impl State { let writer = Mutex::new(()); let db = Arc::new(db); - Ok(Self { db, block_store, inner, writer }) + // Initialize empty SmtForest infrastructure. + // The forest will be populated incrementally as new blocks are imported. + // On startup, the forest is empty and queries will use database reconstruction. + // As blocks are applied, the forest will accumulate recent block data for fast queries. + let storage_forest = RwLock::new(SmtForest::new()); + let storage_roots = RwLock::new(BTreeMap::new()); + let vault_roots = RwLock::new(BTreeMap::new()); + + Ok(Self { + db, + block_store, + inner, + writer, + storage_forest, + storage_roots, + vault_roots, + }) } /// Apply changes of a new block to the DB and in-memory data structures. @@ -367,6 +396,16 @@ impl State { // Signals the write lock has been acquired, and the transaction can be committed let (inform_acquire_done, acquire_done) = oneshot::channel::<()>(); + // Extract account IDs before block is moved into async task + // We'll need these later to populate the SmtForest + let updated_account_ids = Vec::::from_iter( + block + .body() + .updated_accounts() + .iter() + .map(miden_objects::block::BlockAccountUpdate::account_id), + ); + // The DB and in-memory state updates need to be synchronized and are partially // overlapping. Namely, the DB transaction only proceeds after this task acquires the // in-memory write lock. This requires the DB update to run concurrently, so a new task is @@ -426,11 +465,328 @@ impl State { inner.blockchain.push(block_commitment); } + // STEP 1: After successful DB commit, query updated accounts' storage and populate + // SmtForest + self.update_storage_forest_from_db(updated_account_ids, block_num).await?; + info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); Ok(()) } + /// Updates `SmtForest` after a block is successfully applied + /// + /// STEP 1: Query updated accounts' full storage from DB after successful commit + /// + /// This is called after the DB transaction commits successfully, so we can safely + /// query the newly committed storage data. + #[allow(clippy::too_many_lines)] // Complex multi-step process (Steps 1-5) + async fn update_storage_forest_from_db( + &self, + account_ids: Vec, + block_num: BlockNumber, + ) -> Result<(), ApplyBlockError> { + use miden_objects::crypto::merkle::{EmptySubtreeRoots, SMT_DEPTH}; + + if account_ids.is_empty() { + return Ok(()); + } + + tracing::debug!( + target: COMPONENT, + %block_num, + num_accounts = account_ids.len(), + "Querying account storage from DB to populate SmtForest" + ); + + // Query full storage for each updated account at this block + let mut account_storages = Vec::new(); + for &account_id in &account_ids { + match self.db.select_account_storage_at_block(account_id, block_num).await { + Ok(storage) => { + account_storages.push((account_id, storage)); + }, + Err(e) => { + // Log error but don't fail the entire block application + // Forest will be missing this account but DB queries still work + tracing::warn!( + target: COMPONENT, + %account_id, + %block_num, + error = %e, + "Failed to query account storage for SmtForest update" + ); + }, + } + } + + tracing::info!( + target: COMPONENT, + %block_num, + num_accounts = account_storages.len(), + "Successfully queried account storage from DB (Step 1 complete)" + ); + + // STEP 2: Extract Map slots and their entries from account_storages + let mut map_slots_to_populate = Vec::new(); + + for (account_id, storage) in &account_storages { + // Iterate through each slot in the account storage + for (slot_idx, slot) in storage.slots().iter().enumerate() { + let slot_idx_u8 = slot_idx as u8; + + // Only process Map-type slots + if let miden_objects::account::StorageSlot::Map(storage_map) = slot { + // Extract all (key, value) entries from this StorageMap + let entries: Vec<_> = storage_map.entries().collect(); + + tracing::debug!( + target: COMPONENT, + %account_id, + slot_index = slot_idx_u8, + num_entries = entries.len(), + "Extracted Map slot entries" + ); + + map_slots_to_populate.push((*account_id, slot_idx_u8, entries)); + } + } + } + + tracing::info!( + target: COMPONENT, + %block_num, + num_map_slots = map_slots_to_populate.len(), + "Successfully extracted Map slots and entries (Step 2 complete)" + ); + + // STEP 3: Get previous roots from storage_roots or use empty root + let storage_roots = self.storage_roots.read().await; + let prev_block_num = if block_num.as_u32() > 0 { + BlockNumber::from(block_num.as_u32() - 1) + } else { + // Genesis block - no previous block + block_num + }; + + // For each map slot, get the previous root or use empty root + let mut slots_with_prev_roots = Vec::new(); + + for (account_id, slot_idx, entries) in map_slots_to_populate { + // Look up previous root for this (account_id, slot_idx, prev_block) + let prev_root = if block_num.as_u32() > 0 { + storage_roots + .get(&(account_id, slot_idx, prev_block_num)) + .copied() + .unwrap_or_else(|| { + // No previous root found, use empty SMT root + *EmptySubtreeRoots::entry(SMT_DEPTH, 0) + }) + } else { + // Genesis block - use empty root + *EmptySubtreeRoots::entry(SMT_DEPTH, 0) + }; + + tracing::debug!( + target: COMPONENT, + %account_id, + slot_index = slot_idx, + "Retrieved previous root for slot" + ); + + slots_with_prev_roots.push((account_id, slot_idx, prev_root, entries)); + } + + drop(storage_roots); // Release read lock before write operations + + tracing::info!( + target: COMPONENT, + %block_num, + num_slots = slots_with_prev_roots.len(), + "Successfully retrieved previous roots (Step 3 complete)" + ); + + // STEP 4: Use forest.insert(prev_root, key, value) to build new SMTs + let mut forest = self.storage_forest.write().await; + let mut new_roots = Vec::new(); + + for (account_id, slot_idx, prev_root, entries) in slots_with_prev_roots { + // Start with the previous root + let mut current_root = prev_root; + + // Insert all entries into the forest to build the new SMT + for (key, value) in entries { + match forest.insert(current_root, *key, *value) { + Ok(new_root) => { + current_root = new_root; + }, + Err(e) => { + // Log error but continue with other slots + tracing::error!( + target: COMPONENT, + %account_id, + slot_index = slot_idx, + error = ?e, + "Failed to insert entry into SmtForest" + ); + // Skip this slot by breaking out of entry loop + break; + }, + } + } + + // Store the final root after all insertions + new_roots.push((account_id, slot_idx, current_root)); + + tracing::debug!( + target: COMPONENT, + %account_id, + slot_index = slot_idx, + "Built new SMT in forest" + ); + } + + drop(forest); // Release write lock before next write + + tracing::info!( + target: COMPONENT, + %block_num, + num_new_roots = new_roots.len(), + "Successfully built new SMTs in forest (Step 4 complete)" + ); + + // STEP 5: Track new roots in storage_roots map + let mut storage_roots = self.storage_roots.write().await; + + for (account_id, slot_idx, new_root) in new_roots { + // Insert the new root for this (account_id, slot_idx, block_num) triple + storage_roots.insert((account_id, slot_idx, block_num), new_root); + + tracing::debug!( + target: COMPONENT, + %account_id, + slot_index = slot_idx, + %block_num, + "Tracked new root in storage_roots map" + ); + } + + tracing::info!( + target: COMPONENT, + %block_num, + total_tracked_roots = storage_roots.len(), + "Successfully tracked new roots (Step 5 complete)" + ); + + // VAULT TRACKING: Track vault SMT roots for structural sharing + tracing::debug!( + target: COMPONENT, + %block_num, + "Starting vault tracking" + ); + + // Query vault assets for each updated account + let mut vault_entries_to_populate = Vec::new(); + + for &account_id in &account_ids { + match self.db.select_account_vault_at_block(account_id, block_num).await { + Ok(entries) if !entries.is_empty() => { + vault_entries_to_populate.push((account_id, entries)); + }, + Ok(_) => { + tracing::debug!(%account_id, "Account has empty vault"); + }, + Err(e) => { + tracing::warn!(%account_id, error = %e, "Failed to query vault assets"); + }, + } + } + + if vault_entries_to_populate.is_empty() { + tracing::debug!("No vaults to populate"); + return Ok(()); + } + + tracing::info!( + target: COMPONENT, + num_vaults = vault_entries_to_populate.len(), + "Queried vault assets" + ); + + // Get previous vault roots + let vault_roots_read = self.vault_roots.read().await; + let prev_block_num = if block_num.as_u32() > 0 { + BlockNumber::from(block_num.as_u32() - 1) + } else { + block_num + }; + + let mut vaults_with_prev_roots = Vec::new(); + for (account_id, entries) in vault_entries_to_populate { + let prev_root = if block_num.as_u32() > 0 { + vault_roots_read + .get(&(account_id, prev_block_num)) + .copied() + .unwrap_or_else(|| *EmptySubtreeRoots::entry(SMT_DEPTH, 0)) + } else { + *EmptySubtreeRoots::entry(SMT_DEPTH, 0) + }; + + vaults_with_prev_roots.push((account_id, prev_root, entries)); + } + drop(vault_roots_read); + + // Build vault SMTs in forest + let mut forest = self.storage_forest.write().await; + let mut vault_new_roots = Vec::new(); + + for (account_id, prev_root, entries) in vaults_with_prev_roots { + let mut current_root = prev_root; + + for (key, value) in entries { + match forest.insert(current_root, key, value) { + Ok(new_root) => { + current_root = new_root; + }, + Err(e) => { + tracing::error!( + target: COMPONENT, + %account_id, + error = ?e, + "Failed to insert vault entry into SmtForest" + ); + break; + }, + } + } + + vault_new_roots.push((account_id, current_root)); + } + drop(forest); + + tracing::info!( + target: COMPONENT, + %block_num, + num_vault_roots = vault_new_roots.len(), + "Built vault SMTs in forest" + ); + + // Track vault roots + let mut vault_roots = self.vault_roots.write().await; + for (account_id, new_root) in vault_new_roots { + vault_roots.insert((account_id, block_num), new_root); + } + + tracing::info!( + target: COMPONENT, + %block_num, + total_vault_roots = vault_roots.len(), + "Successfully tracked vault roots (Vault tracking complete)" + ); + + Ok(()) + } + /// Queries a [BlockHeader] from the database, and returns it alongside its inclusion proof. /// /// If [None] is given as the value of `block_num`, the data for the latest [BlockHeader] is From 19164bef25e12165efeb91a82d9a1f8c1f22b3a4 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 1 Dec 2025 17:44:32 +0100 Subject: [PATCH 06/95] changset, should go away after rebase --- crates/store/src/state.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index caad0c91a6..d4c9c66649 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -354,7 +354,14 @@ impl State { }; // build note tree - let note_tree = block.body().compute_block_note_tree(); + let note_tree_entries: Vec<_> = block + .body() + .output_notes() + .map(|(note_index, note)| (note_index, note.id(), *note.metadata())) + .collect(); + let note_tree = + miden_objects::block::BlockNoteTree::with_entries(note_tree_entries.iter().copied()) + .map_err(|e| InvalidBlockError::FailedToBuildNoteTree(e.to_string()))?; if note_tree.root() != header.note_root() { return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); } From 8eb49af5dde9aa3d6c3805def9903203ab06d0df Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 1 Dec 2025 19:50:46 +0100 Subject: [PATCH 07/95] improve --- crates/proto/src/domain/account.rs | 17 +++ crates/store/src/state.rs | 174 ++++++++++++++++++++++------- 2 files changed, 152 insertions(+), 39 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index f745c74488..2ee1c658e5 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -472,6 +472,23 @@ impl AccountStorageMapDetails { } } + /// Creates an AccountStorageMapDetails from already-queried entries (e.g., from database). + /// This is useful when entries have been fetched directly rather than extracted from a StorageMap. + pub fn from_entries(slot_index: u8, map_entries: Vec<(Word, Word)>) -> Self { + let too_many_entries = map_entries.len() > Self::MAX_RETURN_ENTRIES; + let map_entries = if too_many_entries { + Vec::new() + } else { + map_entries + }; + + Self { + slot_index, + too_many_entries, + map_entries, + } + } + pub fn too_many_entries(slot_index: u8) -> Self { Self { slot_index, diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index d4c9c66649..d7c66135a9 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -1375,16 +1375,110 @@ impl State { storage_requests, } = detail_request; + // First, get the account summary without deserializing the full account let account_info = self.db.select_historical_account_at(account_id, block_num).await?; - // If we get a query for a public account but the details are missing from the database, - // it indicates an inconsistent state in the database. + // Ensure we have account details (only available for public accounts) let Some(account) = account_info.details else { - return Err(DatabaseError::AccountDetailsMissing(account_id)); + return Err(DatabaseError::AccountNotPublic(account_id)); }; + // Determine if we need to deserialize the full account + // We need it if: + // - We need to return code and the commitment doesn't match + // - We need to return vault data and the commitment doesn't match or is missing + // - We need to return all entries for any storage slot + let need_full_account = code_commitment.is_some() + || asset_vault_commitment.is_some() + || storage_requests.iter().any(|req| matches!(req.slot_data, SlotData::All)); + + if need_full_account { + self.fetch_full_account_details( + account, + code_commitment, + asset_vault_commitment, + storage_requests, + ) + } else { + self.fetch_optimized_account_details( + account, + account_id, + block_num, + storage_requests, + ) + .await + } + } + + /// Fetches full account details when full deserialization is required + fn fetch_full_account_details( + &self, + account: miden_objects::account::Account, + code_commitment: Option, + asset_vault_commitment: Option, + storage_requests: Vec, + ) -> Result { let storage_header = account.storage().to_header(); + let storage_map_details = self.process_storage_map_requests_full(&account, storage_requests)?; + + // Only include account code if the commitment doesn't match + let account_code = code_commitment + .filter(|commitment| *commitment != account.code().commitment()) + .map(|_| account.code().to_bytes()); + + // Handle vault details based on the provided commitment + let vault_details = match asset_vault_commitment { + Some(commitment) if commitment == account.vault().root() => { + AccountVaultDetails::empty() + }, + Some(_) => AccountVaultDetails::new(account.vault()), + None => AccountVaultDetails::empty(), + }; + + let account_header = AccountHeader::from(&account); + + Ok(AccountDetails { + account_header, + account_code, + vault_details, + storage_details: AccountStorageDetails { + header: storage_header, + map_details: storage_map_details, + }, + }) + } + + /// Fetches optimized account details by querying specific keys from DB + async fn fetch_optimized_account_details( + &self, + account: miden_objects::account::Account, + account_id: AccountId, + block_num: BlockNumber, + storage_requests: Vec, + ) -> Result { + let storage_header = account.storage().to_header(); + let storage_map_details = self + .process_storage_map_requests_optimized(account_id, block_num, storage_requests) + .await?; + let account_header = AccountHeader::from(&account); + + Ok(AccountDetails { + account_header, + account_code: None, + vault_details: AccountVaultDetails::empty(), + storage_details: AccountStorageDetails { + header: storage_header, + map_details: storage_map_details, + }, + }) + } + /// Processes storage map requests using full account data + fn process_storage_map_requests_full( + &self, + account: &miden_objects::account::Account, + storage_requests: Vec, + ) -> Result, DatabaseError> { let mut storage_map_details = Vec::::with_capacity(storage_requests.len()); @@ -1398,45 +1492,47 @@ impl State { storage_map_details.push(details); } - // Only include unknown account code blobs, which is equal to a account code digest - // mismatch. If `None` was requested, don't return any. - let account_code = code_commitment - .is_some_and(|code_commitment| code_commitment != account.code().commitment()) - .then(|| account.code().to_bytes()); + Ok(storage_map_details) + } - // storage details - let storage_details = AccountStorageDetails { - header: storage_header, - map_details: storage_map_details, - }; + /// Processes storage map requests by querying DB for specific keys + async fn process_storage_map_requests_optimized( + &self, + account_id: AccountId, + block_num: BlockNumber, + storage_requests: Vec, + ) -> Result, DatabaseError> { + let mut storage_map_details = + Vec::::with_capacity(storage_requests.len()); - // Handle vault details based on the `asset_vault_commitment`. - // Similar to `code_commitment`, if the provided commitment matches, we don't return - // vault data. If no commitment is provided or it doesn't match, we return - // the vault data. If the number of vault contained assets are exceeding a - // limit, we signal this back in the response and the user must handle that - // in follow-up request. - let vault_details = match asset_vault_commitment { - Some(commitment) if commitment == account.vault().root() => { - // The client already has the correct vault data - AccountVaultDetails::empty() - }, - Some(_) => { - // The commitment doesn't match, so return vault data - AccountVaultDetails::new(account.vault()) - }, - None => { - // No commitment provided, so don't return vault data - AccountVaultDetails::empty() - }, - }; + for StorageMapRequest { slot_index, slot_data } in storage_requests { + let details = match slot_data { + SlotData::MapKeys(keys) => { + // Efficiently query specific keys from the DB + let map_entries = self + .db + .select_storage_map_keys_at_block( + account_id, + block_num, + slot_index, + keys.clone(), + ) + .await?; + + AccountStorageMapDetails::from_entries(slot_index, map_entries) + }, + SlotData::All => { + // This should not happen as we check for it in need_full_account + return Err(DatabaseError::DataCorrupted( + "SlotData::All should have been handled in need_full_account check" + .to_string(), + )); + }, + }; + storage_map_details.push(details); + } - Ok(AccountDetails { - account_header: AccountHeader::from(account), - account_code, - vault_details, - storage_details, - }) + Ok(storage_map_details) } /// Returns storage map values for syncing within a block range. From 6725461bee0f544f5eba390feaea63e96f527447 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 1 Dec 2025 23:56:55 +0100 Subject: [PATCH 08/95] TODO and deprecation --- crates/proto/src/domain/account.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 2ee1c658e5..2668158e23 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -69,6 +69,7 @@ impl From for proto::account::AccountId { // ACCOUNT UPDATE // ================================================================================================ +// TODO should be called `AccountStateRef` or so #[derive(Debug, PartialEq)] pub struct AccountSummary { pub account_id: AccountId, @@ -86,6 +87,7 @@ impl From<&AccountSummary> for proto::account::AccountSummary { } } +#[deprecated(note = "avoid this type, details will be `None` always!")] #[derive(Debug, PartialEq)] pub struct AccountInfo { pub summary: AccountSummary, From ee65a8857e31ba4eda82cef1baff948e6f3fd845 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 1 Dec 2025 23:58:28 +0100 Subject: [PATCH 09/95] account queries --- crates/proto/src/domain/account.rs | 9 +- .../store/src/db/models/queries/accounts.rs | 357 +++++++++--------- crates/store/src/state.rs | 134 +++++-- 3 files changed, 283 insertions(+), 217 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 2668158e23..7dc8737ba5 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -475,14 +475,11 @@ impl AccountStorageMapDetails { } /// Creates an AccountStorageMapDetails from already-queried entries (e.g., from database). - /// This is useful when entries have been fetched directly rather than extracted from a StorageMap. + /// This is useful when entries have been fetched directly rather than extracted from a + /// StorageMap. pub fn from_entries(slot_index: u8, map_entries: Vec<(Word, Word)>) -> Self { let too_many_entries = map_entries.len() > Self::MAX_RETURN_ENTRIES; - let map_entries = if too_many_entries { - Vec::new() - } else { - map_entries - }; + let map_entries = if too_many_entries { Vec::new() } else { map_entries }; Self { slot_index, diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index a0e79e917b..87a231996e 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -41,12 +41,18 @@ use crate::db::models::{serialize_vec, vec_raw_try_into}; use crate::db::{AccountVaultValue, schema}; use crate::errors::DatabaseError; -/// Select the latest account details by account id from the DB using the given +/// Select the latest account info by account id from the DB using the given /// [`SqliteConnection`]. /// /// # Returns /// -/// The latest account details, or an error. +/// The latest account info, or an error. +/// +/// # Note +/// +/// Returns only the account summary. Full account details must be reconstructed +/// in follow up query, using separate query functions to fetch specific account +/// components as needed. /// /// # Raw SQL /// @@ -54,16 +60,9 @@ use crate::errors::DatabaseError; /// SELECT /// accounts.account_id, /// accounts.account_commitment, -/// accounts.block_num, -/// accounts.storage, -/// accounts.vault, -/// accounts.nonce, -/// accounts.code_commitment, -/// account_codes.code +/// accounts.block_num /// FROM /// accounts -/// LEFT JOIN -/// account_codes ON accounts.code_commitment = account_codes.code_commitment /// WHERE /// account_id = ?1 /// AND is_latest = 1 @@ -72,27 +71,38 @@ pub(crate) fn select_account( conn: &mut SqliteConnection, account_id: AccountId, ) -> Result { - let raw = SelectDsl::select( - schema::accounts::table.left_join(schema::account_codes::table.on( - schema::accounts::code_commitment.eq(schema::account_codes::code_commitment.nullable()), - )), - (AccountRaw::as_select(), schema::account_codes::code.nullable()), - ) - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::is_latest.eq(true)) - .get_result::<(AccountRaw, Option>)>(conn) - .optional()? - .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; - let info = AccountWithCodeRawJoined::from(raw).try_into()?; - Ok(info) + let raw = SelectDsl::select(schema::accounts::table, AccountSummaryRaw::as_select()) + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .get_result::(conn) + .optional()? + .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + + let summary: AccountSummary = raw.try_into()?; + + // Backfill account details from database + // For private accounts, we don't store full details in the database + let details = if account_id.is_public() { + Some(reconstruct_full_account_from_db(conn, account_id)?) + } else { + None + }; + + Ok(AccountInfo { summary, details }) } -/// Select account details at a specific block number from the DB using the given +/// Select account info at a specific block number from the DB using the given /// [`SqliteConnection`]. /// /// # Returns /// -/// The account details at the specified block, or an error. +/// The account info at the specified block, or an error. +/// +/// # Note +/// +/// This function returns only the account summary (id, commitment, `block_num`). +/// Full account details are no longer reconstructed here - use separate query functions +/// to fetch specific account components as needed. /// /// # Raw SQL /// @@ -100,16 +110,9 @@ pub(crate) fn select_account( /// SELECT /// accounts.account_id, /// accounts.account_commitment, -/// accounts.block_num, -/// accounts.storage, -/// accounts.vault, -/// accounts.nonce, -/// accounts.code_commitment, -/// account_codes.code +/// accounts.block_num /// FROM /// accounts -/// LEFT JOIN -/// account_codes ON accounts.code_commitment = account_codes.code_commitment /// WHERE /// account_id = ?1 /// AND block_num = ?2 @@ -119,33 +122,34 @@ pub(crate) fn select_historical_account_at( account_id: AccountId, block_num: BlockNumber, ) -> Result { - let raw = SelectDsl::select( - schema::accounts::table.left_join(schema::account_codes::table.on( - schema::accounts::code_commitment.eq(schema::account_codes::code_commitment.nullable()), - )), - (AccountRaw::as_select(), schema::account_codes::code.nullable()), - ) - .filter( - schema::accounts::account_id - .eq(account_id.to_bytes()) - .and(schema::accounts::block_num.eq(block_num.to_raw_sql())), - ) - .get_result::<(AccountRaw, Option>)>(conn) - .optional()? - .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; - let info = AccountWithCodeRawJoined::from(raw).try_into()?; - Ok(info) + let raw = SelectDsl::select(schema::accounts::table, AccountSummaryRaw::as_select()) + .filter( + schema::accounts::account_id + .eq(account_id.to_bytes()) + .and(schema::accounts::block_num.eq(block_num.to_raw_sql())), + ) + .get_result::(conn) + .optional()? + .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + + let summary: AccountSummary = raw.try_into()?; + + // Backfill account details from database (at the specific historical block) + // Note: We use `ok()` to convert errors to None, as historical data might not have full details + let details = reconstruct_full_account_from_db(conn, account_id).ok(); + + Ok(AccountInfo { summary, details }) } -/// Select the latest account details by account ID prefix from the DB using the given -/// [`SqliteConnection`] This method is meant to be used by the network transaction builder. Because -/// network notes get matched through accounts through the account's 30-bit prefix, it is possible -/// that multiple accounts match against a single prefix. In this scenario, the first account is -/// returned. +/// Select the latest account info by account ID prefix from the DB using the given +/// [`SqliteConnection`]. This method is meant to be used by the network transaction builder. +/// Because network notes get matched through accounts through the account's 30-bit prefix, it is +/// possible that multiple accounts match against a single prefix. In this scenario, the first +/// account is returned. /// /// # Returns /// -/// The latest account details, `None` if the account was not found, or an error. +/// The latest account info, `None` if the account was not found, or an error. /// /// # Raw SQL /// @@ -153,41 +157,34 @@ pub(crate) fn select_historical_account_at( /// SELECT /// accounts.account_id, /// accounts.account_commitment, -/// accounts.block_num, -/// accounts.storage, -/// accounts.vault, -/// accounts.nonce, -/// accounts.code_commitment, -/// account_codes.code +/// accounts.block_num /// FROM /// accounts -/// LEFT JOIN -/// account_codes ON accounts.code_commitment = account_codes.code_commitment /// WHERE /// network_account_id_prefix = ?1 +/// AND is_latest = 1 /// ``` pub(crate) fn select_account_by_id_prefix( conn: &mut SqliteConnection, id_prefix: u32, ) -> Result, DatabaseError> { - let maybe_info = SelectDsl::select( - schema::accounts::table.left_join(schema::account_codes::table.on( - schema::accounts::code_commitment.eq(schema::account_codes::code_commitment.nullable()), - )), - (AccountRaw::as_select(), schema::account_codes::code.nullable()), - ) - .filter(schema::accounts::is_latest.eq(true)) - .filter(schema::accounts::network_account_id_prefix.eq(Some(i64::from(id_prefix)))) - .get_result::<(AccountRaw, Option>)>(conn) - .optional() - .map_err(DatabaseError::Diesel)?; - - let result: Result, DatabaseError> = maybe_info - .map(AccountWithCodeRawJoined::from) - .map(std::convert::TryInto::::try_into) - .transpose(); - - result + let maybe_summary = SelectDsl::select(schema::accounts::table, AccountSummaryRaw::as_select()) + .filter(schema::accounts::is_latest.eq(true)) + .filter(schema::accounts::network_account_id_prefix.eq(Some(i64::from(id_prefix)))) + .get_result::(conn) + .optional() + .map_err(DatabaseError::Diesel)?; + + match maybe_summary { + None => Ok(None), + Some(raw) => { + let summary: AccountSummary = raw.try_into()?; + let account_id = summary.account_id; + // Backfill account details from database + let details = reconstruct_full_account_from_db(conn, account_id).ok(); + Ok(Some(AccountInfo { summary, details })) + }, + } } /// Select all account commitments from the DB using the given [`SqliteConnection`]. @@ -368,16 +365,11 @@ pub fn select_accounts_by_block_range( /// SELECT /// accounts.account_id, /// accounts.account_commitment, -/// accounts.block_num, -/// accounts.storage, -/// accounts.vault, -/// accounts.nonce, -/// accounts.code_commitment, -/// account_codes.code +/// accounts.block_num /// FROM /// accounts -/// LEFT JOIN -/// account_codes ON accounts.code_commitment = account_codes.code_commitment +/// WHERE +/// is_latest = 1 /// ORDER BY /// block_num ASC /// ``` @@ -385,17 +377,23 @@ pub fn select_accounts_by_block_range( pub(crate) fn select_all_accounts( conn: &mut SqliteConnection, ) -> Result, DatabaseError> { - let accounts_raw = QueryDsl::select( - schema::accounts::table.left_join(schema::account_codes::table.on( - schema::accounts::code_commitment.eq(schema::account_codes::code_commitment.nullable()), - )), - (AccountRaw::as_select(), schema::account_codes::code.nullable()), - ) - .filter(schema::accounts::is_latest.eq(true)) - .load::<(AccountRaw, Option>)>(conn)?; - let account_infos = vec_raw_try_into::( - accounts_raw.into_iter().map(AccountWithCodeRawJoined::from), - )?; + let raw = SelectDsl::select(schema::accounts::table, AccountSummaryRaw::as_select()) + .filter(schema::accounts::is_latest.eq(true)) + .order_by(schema::accounts::block_num.asc()) + .load::(conn)?; + + let summaries: Vec = vec_raw_try_into(raw).unwrap(); + + // Backfill account details from database + let account_infos = summaries + .into_iter() + .map(|summary| { + let account_id = summary.account_id; + let details = reconstruct_full_account_from_db(conn, account_id).ok(); + AccountInfo { summary, details } + }) + .collect(); + Ok(account_infos) } @@ -868,59 +866,6 @@ pub struct AccountRaw { pub nonce: Option, } -#[derive(Debug, Clone, QueryableByName)] -pub struct AccountWithCodeRawJoined { - #[diesel(embed)] - pub account: AccountRaw, - #[diesel(embed)] - pub code: Option>, -} - -impl From<(AccountRaw, Option>)> for AccountWithCodeRawJoined { - fn from((account, code): (AccountRaw, Option>)) -> Self { - Self { account, code } - } -} - -impl TryInto for AccountWithCodeRawJoined { - type Error = DatabaseError; - fn try_into(self) -> Result { - use proto::domain::account::{AccountInfo, AccountSummary}; - - let account_id = AccountId::read_from_bytes(&self.account.account_id[..])?; - let account_commitment = Word::read_from_bytes(&self.account.account_commitment[..])?; - let block_num = BlockNumber::from_raw_sql(self.account.block_num)?; - let summary = AccountSummary { - account_id, - account_commitment, - block_num, - }; - let maybe_account = self.try_into()?; - Ok(AccountInfo { summary, details: maybe_account }) - } -} - -impl TryInto> for AccountWithCodeRawJoined { - type Error = DatabaseError; - fn try_into(self) -> Result, Self::Error> { - let _account_id = AccountId::read_from_bytes(&self.account.account_id[..])?; - - // TODO: Storage and vault reconstruction needs database connection - // This implementation is incomplete - it returns None for now - // The proper fix is to refactor account loading at higher level - // where we have access to the connection to call select_account_storage_at_block() - // and select_account_vault_at_block() - let details = if let (Some(_nonce), Some(_code)) = (self.account.nonce, self.code) { - // For now, return None since we can't reconstruct storage/vault without DB connection - // This needs architectural changes in how accounts are loaded - None - } else { - // a private account - None - }; - Ok(details) - } -} #[derive(Debug, Clone, PartialEq, Eq, Selectable, Queryable, QueryableByName)] #[diesel(table_name = schema::accounts)] @@ -1087,6 +1032,80 @@ pub(crate) fn insert_account_storage_map_value( Ok(update_count + insert_count) } +/// Reconstruct full Account from database tables for the latest account state +/// +/// This function queries the database tables to reconstruct a complete Account object: +/// - Code from `account_codes` table +/// - Nonce from `accounts` table +/// - Storage from `account_storage_headers` and `account_storage_map_values` tables +/// - Vault from `account_vault_assets` table +/// +/// # Note +/// +/// This is used by `upsert_accounts` when applying deltas. In the future, this should +/// be replaced with reconstruction from SmtForest state. +fn reconstruct_full_account_from_db( + conn: &mut SqliteConnection, + account_id: AccountId, +) -> Result { + // Get account metadata (nonce, code_commitment) + let account_raw = SelectDsl::select(schema::accounts::table, AccountRaw::as_select()) + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .get_result::(conn) + .optional()? + .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + + let nonce_val = account_raw.nonce.ok_or_else(|| { + DatabaseError::DataCorrupted(format!("No nonce found for account {account_id}")) + })?; + let nonce = Nonce::try_from(u64::try_from(nonce_val).map_err(|_| { + DatabaseError::DataCorrupted(format!("Invalid nonce value for account {account_id}")) + })?)?; + + // Get account code + let code_commitment_bytes = schema::accounts::table + .select(schema::accounts::code_commitment) + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .get_result::>>(conn)? + .ok_or_else(|| { + DatabaseError::DataCorrupted(format!( + "No code commitment found for account {account_id}" + )) + })?; + + let code_bytes = schema::account_codes::table + .select(schema::account_codes::code) + .filter(schema::account_codes::code_commitment.eq(&code_commitment_bytes)) + .get_result::>(conn)?; + + let code = AccountCode::read_from_bytes(&code_bytes)?; + + // Reconstruct storage using existing helper function + let storage = select_latest_account_storage(conn, account_id)?; + + // Reconstruct vault from account_vault_assets table + let vault_entries: Vec<(Vec, Option>)> = schema::account_vault_assets::table + .select((schema::account_vault_assets::vault_key, schema::account_vault_assets::asset)) + .filter(schema::account_vault_assets::account_id.eq(account_id.to_bytes())) + .filter(schema::account_vault_assets::is_latest.eq(true)) + .load(conn)?; + + let mut assets = Vec::new(); + for (key_bytes, maybe_asset_bytes) in vault_entries { + if let Some(asset_bytes) = maybe_asset_bytes { + let asset = Asset::read_from_bytes(&asset_bytes)?; + assets.push(asset); + } + } + + let vault = AssetVault::new(&assets)?; + + // Construct the full account + Account::from_parts(account_id, vault, storage, code, nonce) +} + /// Attention: Assumes the account details are NOT null! The schema explicitly allows this though! #[allow(clippy::too_many_lines)] pub(crate) fn upsert_accounts( @@ -1096,32 +1115,6 @@ pub(crate) fn upsert_accounts( ) -> Result { use proto::domain::account::NetworkAccountPrefix; - fn select_details_stmt( - conn: &mut SqliteConnection, - account_id: AccountId, - ) -> Result, DatabaseError> { - let account_id = account_id.to_bytes(); - let accounts = SelectDsl::select( - schema::accounts::table.left_join( - schema::account_codes::table.on(schema::accounts::code_commitment - .eq(schema::account_codes::code_commitment.nullable())), - ), - (AccountRaw::as_select(), schema::account_codes::code.nullable()), - ) - .filter(schema::accounts::account_id.eq(account_id)) - .filter(schema::accounts::is_latest.eq(true)) - .get_results::<(AccountRaw, Option>)>(conn)?; - - // SELECT .. FROM accounts LEFT JOIN account_codes - // ON accounts.code_commitment == account_codes.code_commitment - - let accounts = Result::from_iter(accounts.into_iter().filter_map(|x| { - let account_with_code = AccountWithCodeRawJoined::from(x); - account_with_code.try_into().transpose() - }))?; - Ok(accounts) - } - let mut count = 0; for update in accounts { let account_id = update.account_id(); @@ -1169,10 +1162,8 @@ pub(crate) fn upsert_accounts( Some(account) }, AccountUpdateDetails::Delta(delta) => { - let mut rows = select_details_stmt(conn, account_id)?.into_iter(); - let Some(account) = rows.next() else { - return Err(DatabaseError::AccountNotFoundInDb(account_id)); - }; + // Reconstruct the full account from database tables + let account = reconstruct_full_account_from_db(conn, account_id)?; // --- process storage map updates ---------------------------- diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index d7c66135a9..6884f875db 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -1317,7 +1317,7 @@ impl State { let (block_num, witness) = self.get_block_witness(block_num, account_id).await?; let details = if let Some(request) = details { - Some(self.fetch_account_proof_details(account_id, block_num, request).await?) + Some(self.fetch_requested_account_details(account_id, block_num, request).await?) } else { None }; @@ -1363,7 +1363,7 @@ impl State { /// /// This method queries the database to fetch the account state and processes the detail /// request to return only the requested information. - async fn fetch_account_proof_details( + async fn fetch_requested_account_details( &self, account_id: AccountId, block_num: BlockNumber, @@ -1382,31 +1382,102 @@ impl State { let Some(account) = account_info.details else { return Err(DatabaseError::AccountNotPublic(account_id)); }; + let need_vault_from_account = asset_vault_commitment.is_some(); + let need_full_storage_maps_from_account = + storage_requests.iter().any(|req| matches!(req.slot_data, SlotData::All)); - // Determine if we need to deserialize the full account - // We need it if: - // - We need to return code and the commitment doesn't match - // - We need to return vault data and the commitment doesn't match or is missing - // - We need to return all entries for any storage slot - let need_full_account = code_commitment.is_some() - || asset_vault_commitment.is_some() - || storage_requests.iter().any(|req| matches!(req.slot_data, SlotData::All)); + let need_full_account = need_vault_from_account && need_full_storage_maps_from_account; if need_full_account { - self.fetch_full_account_details( - account, - code_commitment, - asset_vault_commitment, - storage_requests, - ) + // Inlined fetch_full_account_details + let storage_header = account.storage().to_header(); + let mut storage_map_details = + Vec::::with_capacity(storage_requests.len()); + + for StorageMapRequest { slot_index, slot_data } in storage_requests { + let Some(StorageSlot::Map(storage_map)) = + account.storage().slots().get(slot_index as usize) + else { + return Err(AccountError::StorageSlotNotMap(slot_index).into()); + }; + let details = AccountStorageMapDetails::new(slot_index, slot_data, storage_map); + storage_map_details.push(details); + } + + // Only include account code if the commitment doesn't match + let account_code = code_commitment + .filter(|commitment| *commitment != account.code().commitment()) + .map(|_| account.code().to_bytes()); + + // Handle vault details based on the provided commitment + let vault_details = match asset_vault_commitment { + Some(commitment) if commitment == account.vault().root() => { + AccountVaultDetails::empty() + }, + Some(_) => AccountVaultDetails::new(account.vault()), + None => AccountVaultDetails::empty(), + }; + + let account_header = AccountHeader::from(&account); + + Ok(AccountDetails { + account_header, + account_code, + vault_details, + storage_details: AccountStorageDetails { + header: storage_header, + map_details: storage_map_details, + }, + }) } else { - self.fetch_optimized_account_details( - account, - account_id, - block_num, - storage_requests, - ) - .await + // Inlined fetch_optimized_account_details + let storage_header = account.storage().to_header(); + let mut storage_map_details = + Vec::::with_capacity(storage_requests.len()); + + for StorageMapRequest { slot_index, slot_data } in storage_requests { + let details = match slot_data { + SlotData::MapKeys(keys) => { + // Efficiently query specific keys from the DB + let map_entries = self + .db + .select_storage_map_keys_at_block( + account_id, + block_num, + slot_index, + keys.clone(), + ) + .await?; + + AccountStorageMapDetails::from_entries(slot_index, map_entries) + }, + SlotData::All => { + // This should not happen as we check for it in need_full_account + return Err(DatabaseError::DataCorrupted( + "SlotData::All should have been handled in need_full_account check" + .to_string(), + )); + }, + }; + storage_map_details.push(details); + } + + let account_header = AccountHeader::from(&account); + + // Only include account code if the commitment doesn't match + let account_code = code_commitment + .filter(|commitment| *commitment != account.code().commitment()) + .map(|_| account.code().to_bytes()); + + Ok(AccountDetails { + account_header, + account_code, + vault_details: AccountVaultDetails::empty(), + storage_details: AccountStorageDetails { + header: storage_header, + map_details: storage_map_details, + }, + }) } } @@ -1419,13 +1490,14 @@ impl State { storage_requests: Vec, ) -> Result { let storage_header = account.storage().to_header(); - let storage_map_details = self.process_storage_map_requests_full(&account, storage_requests)?; - + let storage_map_details = + self.process_storage_map_requests_full(&account, storage_requests)?; + // Only include account code if the commitment doesn't match let account_code = code_commitment .filter(|commitment| *commitment != account.code().commitment()) .map(|_| account.code().to_bytes()); - + // Handle vault details based on the provided commitment let vault_details = match asset_vault_commitment { Some(commitment) if commitment == account.vault().root() => { @@ -1434,7 +1506,7 @@ impl State { Some(_) => AccountVaultDetails::new(account.vault()), None => AccountVaultDetails::empty(), }; - + let account_header = AccountHeader::from(&account); Ok(AccountDetails { @@ -1454,6 +1526,7 @@ impl State { account: miden_objects::account::Account, account_id: AccountId, block_num: BlockNumber, + code_commitment: Option, storage_requests: Vec, ) -> Result { let storage_header = account.storage().to_header(); @@ -1462,9 +1535,14 @@ impl State { .await?; let account_header = AccountHeader::from(&account); + // Only include account code if the commitment doesn't match + let account_code = code_commitment + .filter(|commitment| *commitment != account.code().commitment()) + .map(|_| account.code().to_bytes()); + Ok(AccountDetails { account_header, - account_code: None, + account_code, vault_details: AccountVaultDetails::empty(), storage_details: AccountStorageDetails { header: storage_header, From 741df6f9418d14e3f39a8c819063fa2815207df9 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 2 Dec 2025 01:29:21 +0100 Subject: [PATCH 10/95] yes --- crates/proto/src/domain/account.rs | 2 +- crates/store/src/db/mod.rs | 8 +- crates/store/src/db/models/conv.rs | 4 +- .../store/src/db/models/queries/accounts.rs | 95 ++-- crates/store/src/state.rs | 426 +++++------------- 5 files changed, 164 insertions(+), 371 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 7dc8737ba5..1e02187865 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -87,7 +87,7 @@ impl From<&AccountSummary> for proto::account::AccountSummary { } } -#[deprecated(note = "avoid this type, details will be `None` always!")] +// TODO #[deprecated(note = "avoid this type, details will be `None` always!")] #[derive(Debug, PartialEq)] pub struct AccountInfo { pub summary: AccountSummary, diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 840c0e2411..0b97ec5f24 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -13,12 +13,7 @@ use miden_objects::asset::{Asset, AssetVaultKey}; use miden_objects::block::{BlockHeader, BlockNoteIndex, BlockNumber, ProvenBlock}; use miden_objects::crypto::merkle::SparseMerklePath; use miden_objects::note::{ - NoteDetails, - NoteId, - NoteInclusionProof, - NoteMetadata, - NoteScript, - Nullifier, + NoteDetails, NoteId, NoteInclusionProof, NoteMetadata, NoteScript, Nullifier, }; use miden_objects::transaction::TransactionId; use tokio::sync::oneshot; @@ -393,6 +388,7 @@ impl Db { } /// Loads all the account commitments from the DB. + // TODO add a variant with block_num as arg #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_account_commitments(&self) -> Result> { self.transact("read all account commitments", move |conn| { diff --git a/crates/store/src/db/models/conv.rs b/crates/store/src/db/models/conv.rs index 858ed59c5c..2aa78a8af1 100644 --- a/crates/store/src/db/models/conv.rs +++ b/crates/store/src/db/models/conv.rs @@ -153,9 +153,9 @@ pub(crate) fn nullifier_prefix_to_raw_sql(prefix: u16) -> i32 { } #[inline(always)] -pub(crate) fn raw_sql_to_nonce(raw: i64) -> u64 { +pub(crate) fn raw_sql_to_nonce(raw: i64) -> Felt { debug_assert!(raw >= 0); - raw as u64 + Felt::new(raw as u64) } #[inline(always)] pub(crate) fn nonce_to_raw_sql(nonce: Felt) -> i64 { diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 87a231996e..7e181530c1 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -4,18 +4,8 @@ use diesel::prelude::{Queryable, QueryableByName}; use diesel::query_dsl::methods::SelectDsl; use diesel::sqlite::Sqlite; use diesel::{ - AsChangeset, - BoolExpressionMethods, - ExpressionMethods, - Insertable, - JoinOnDsl, - NullableExpressionMethods, - OptionalExtension, - QueryDsl, - RunQueryDsl, - Selectable, - SelectableHelper, - SqliteConnection, + AsChangeset, BoolExpressionMethods, ExpressionMethods, Insertable, OptionalExtension, QueryDsl, + RunQueryDsl, Selectable, SelectableHelper, SqliteConnection, }; use miden_lib::utils::{Deserializable, Serializable}; use miden_node_proto as proto; @@ -24,19 +14,16 @@ use miden_node_utils::limiter::{QueryParamAccountIdLimit, QueryParamLimiter}; use miden_objects::Word; use miden_objects::account::delta::AccountUpdateDetails; use miden_objects::account::{ - Account, - AccountDelta, - AccountId, - AccountStorage, - NonFungibleDeltaAction, - StorageSlot, - StorageSlotType, + Account, AccountCode, AccountDelta, AccountId, AccountStorage, NonFungibleDeltaAction, + StorageSlot, StorageSlotType, }; -use miden_objects::asset::{Asset, AssetVaultKey, FungibleAsset}; +use miden_objects::asset::{Asset, AssetVault, AssetVaultKey, FungibleAsset}; use miden_objects::block::{BlockAccountUpdate, BlockNumber}; use crate::constants::MAX_PAYLOAD_BYTES; -use crate::db::models::conv::{SqlTypeConvert, nonce_to_raw_sql, raw_sql_to_slot, slot_to_raw_sql}; +use crate::db::models::conv::{ + SqlTypeConvert, nonce_to_raw_sql, raw_sql_to_nonce, raw_sql_to_slot, slot_to_raw_sql, +}; use crate::db::models::{serialize_vec, vec_raw_try_into}; use crate::db::{AccountVaultValue, schema}; use crate::errors::DatabaseError; @@ -866,7 +853,6 @@ pub struct AccountRaw { pub nonce: Option, } - #[derive(Debug, Clone, PartialEq, Eq, Selectable, Queryable, QueryableByName)] #[diesel(table_name = schema::accounts)] #[diesel(check_for_backend(Sqlite))] @@ -1042,43 +1028,29 @@ pub(crate) fn insert_account_storage_map_value( /// /// # Note /// -/// This is used by `upsert_accounts` when applying deltas. In the future, this should -/// be replaced with reconstruction from SmtForest state. +/// A stop-gap solution to retain store API and construct `AccountInfo` types. +/// The function should ultimately be removed, and any queries be served from the +/// `State` which contains an `SmtForest` to serve the latest and most recent +/// historical data. +// TODO: remove eventually once refactoring is complete fn reconstruct_full_account_from_db( conn: &mut SqliteConnection, account_id: AccountId, ) -> Result { - // Get account metadata (nonce, code_commitment) - let account_raw = SelectDsl::select(schema::accounts::table, AccountRaw::as_select()) - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::is_latest.eq(true)) - .get_result::(conn) - .optional()? - .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + // Get account metadata (nonce, code_commitment) and code in a single join query + let (account_raw, code_bytes): (AccountRaw, Vec) = SelectDsl::select( + schema::accounts::table.inner_join(schema::account_codes::table), + (AccountRaw::as_select(), schema::account_codes::code), + ) + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .get_result(conn) + .optional()? + .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; - let nonce_val = account_raw.nonce.ok_or_else(|| { + let nonce = raw_sql_to_nonce(account_raw.nonce.ok_or_else(|| { DatabaseError::DataCorrupted(format!("No nonce found for account {account_id}")) - })?; - let nonce = Nonce::try_from(u64::try_from(nonce_val).map_err(|_| { - DatabaseError::DataCorrupted(format!("Invalid nonce value for account {account_id}")) - })?)?; - - // Get account code - let code_commitment_bytes = schema::accounts::table - .select(schema::accounts::code_commitment) - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::is_latest.eq(true)) - .get_result::>>(conn)? - .ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "No code commitment found for account {account_id}" - )) - })?; - - let code_bytes = schema::account_codes::table - .select(schema::account_codes::code) - .filter(schema::account_codes::code_commitment.eq(&code_commitment_bytes)) - .get_result::>(conn)?; + })?); let code = AccountCode::read_from_bytes(&code_bytes)?; @@ -1086,14 +1058,16 @@ fn reconstruct_full_account_from_db( let storage = select_latest_account_storage(conn, account_id)?; // Reconstruct vault from account_vault_assets table - let vault_entries: Vec<(Vec, Option>)> = schema::account_vault_assets::table - .select((schema::account_vault_assets::vault_key, schema::account_vault_assets::asset)) - .filter(schema::account_vault_assets::account_id.eq(account_id.to_bytes())) - .filter(schema::account_vault_assets::is_latest.eq(true)) - .load(conn)?; + let vault_entries: Vec<(Vec, Option>)> = SelectDsl::select( + schema::account_vault_assets::table, + (schema::account_vault_assets::vault_key, schema::account_vault_assets::asset), + ) + .filter(schema::account_vault_assets::account_id.eq(account_id.to_bytes())) + .filter(schema::account_vault_assets::is_latest.eq(true)) + .load(conn)?; let mut assets = Vec::new(); - for (key_bytes, maybe_asset_bytes) in vault_entries { + for (_key_bytes, maybe_asset_bytes) in vault_entries { if let Some(asset_bytes) = maybe_asset_bytes { let asset = Asset::read_from_bytes(&asset_bytes)?; assets.push(asset); @@ -1102,8 +1076,7 @@ fn reconstruct_full_account_from_db( let vault = AssetVault::new(&assets)?; - // Construct the full account - Account::from_parts(account_id, vault, storage, code, nonce) + Ok(Account::new(account_id, vault, storage, code, nonce, None)?) } /// Attention: Assumes the account details are NOT null! The schema explicitly allows this though! diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 6884f875db..7037dc6eb4 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -9,17 +9,9 @@ use std::path::Path; use std::sync::Arc; use miden_node_proto::domain::account::{ - AccountDetailRequest, - AccountDetails, - AccountInfo, - AccountProofRequest, - AccountProofResponse, - AccountStorageDetails, - AccountStorageMapDetails, - AccountVaultDetails, - NetworkAccountPrefix, - SlotData, - StorageMapRequest, + AccountDetailRequest, AccountDetails, AccountInfo, AccountProofRequest, AccountProofResponse, + AccountStorageDetails, AccountStorageMapDetails, AccountVaultDetails, NetworkAccountPrefix, + SlotData, StorageMapRequest, }; use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; @@ -28,26 +20,12 @@ use miden_objects::account::{AccountHeader, AccountId, StorageSlot}; use miden_objects::block::account_tree::{AccountTree, account_id_to_smt_key}; use miden_objects::block::nullifier_tree::NullifierTree; use miden_objects::block::{ - AccountWitness, - BlockHeader, - BlockInputs, - BlockNumber, - Blockchain, - NullifierWitness, + AccountWitness, BlockHeader, BlockInputs, BlockNumber, Blockchain, NullifierWitness, ProvenBlock, }; use miden_objects::crypto::merkle::{ - Forest, - LargeSmt, - MemoryStorage, - Mmr, - MmrDelta, - MmrPeaks, - MmrProof, - PartialMmr, - SmtForest, - SmtProof, - SmtStorage, + Forest, LargeSmt, MemoryStorage, Mmr, MmrDelta, MmrPeaks, MmrProof, PartialMmr, SmtForest, + SmtProof, SmtStorage, }; use miden_objects::note::{NoteDetails, NoteId, NoteScript, Nullifier}; use miden_objects::transaction::{OutputNote, PartialBlockchain}; @@ -58,25 +36,13 @@ use tracing::{info, info_span, instrument}; use crate::blocks::BlockStore; use crate::db::models::Page; -use crate::db::models::queries::StorageMapValuesPage; +use crate::db::models::queries::{StorageMapValuesPage, select_account_storage_headers_at_block}; use crate::db::{ - AccountVaultValue, - Db, - NoteRecord, - NoteSyncUpdate, - NullifierInfo, - StateSyncUpdate, + AccountVaultValue, Db, NoteRecord, NoteSyncUpdate, NullifierInfo, StateSyncUpdate, }; use crate::errors::{ - ApplyBlockError, - DatabaseError, - GetBatchInputsError, - GetBlockHeaderError, - GetBlockInputsError, - GetCurrentBlockchainDataError, - InvalidBlockError, - NoteSyncError, - StateInitializationError, + ApplyBlockError, DatabaseError, GetBatchInputsError, GetBlockHeaderError, GetBlockInputsError, + GetCurrentBlockchainDataError, InvalidBlockError, NoteSyncError, StateInitializationError, StateSyncError, }; use crate::{AccountTreeWithHistory, COMPONENT, DataDirectory}; @@ -181,15 +147,11 @@ impl State { let writer = Mutex::new(()); let db = Arc::new(db); - // Initialize empty SmtForest infrastructure. - // The forest will be populated incrementally as new blocks are imported. - // On startup, the forest is empty and queries will use database reconstruction. - // As blocks are applied, the forest will accumulate recent block data for fast queries. let storage_forest = RwLock::new(SmtForest::new()); let storage_roots = RwLock::new(BTreeMap::new()); let vault_roots = RwLock::new(BTreeMap::new()); - Ok(Self { + let me = Self { db, block_store, inner, @@ -197,7 +159,15 @@ impl State { storage_forest, storage_roots, vault_roots, - }) + }; + + // load all accounts from the table + // TODO: make `select_all_account_at(block_num)` to be precise; if ACID is upheld, it's not necessary in theory + let acc_account_ids = db.select_all_account_commitments().await?; + let acc_account_ids = Vec::from_iter(acc_account_ids.into_iter().map(|(account_id, _)| acc_account_ids)); + me.update_storage_forest_from_db(acc_account_ids, latest_block_num)?; + + Ok(me) } /// Apply changes of a new block to the DB and in-memory data structures. @@ -472,8 +442,8 @@ impl State { inner.blockchain.push(block_commitment); } - // STEP 1: After successful DB commit, query updated accounts' storage and populate - // SmtForest + // After successful DB commit, query updated accounts' storage as well as vault data + // TODO look into making this consume the `account_tree_update` self.update_storage_forest_from_db(updated_account_ids, block_num).await?; info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); @@ -483,48 +453,54 @@ impl State { /// Updates `SmtForest` after a block is successfully applied /// - /// STEP 1: Query updated accounts' full storage from DB after successful commit - /// - /// This is called after the DB transaction commits successfully, so we can safely + /// Must be called after the DB transaction commits successfully, so we can safely /// query the newly committed storage data. - #[allow(clippy::too_many_lines)] // Complex multi-step process (Steps 1-5) + /// + /// # Warning + /// + /// Has internal locking to mutate the state, use cautiously in scopes with other + /// mutex guards around! + /// + /// # Note + /// + /// The number of changed accounts is bounded by transactions per block. async fn update_storage_forest_from_db( &self, - account_ids: Vec, + changed_account_ids: Vec, block_num: BlockNumber, ) -> Result<(), ApplyBlockError> { - use miden_objects::crypto::merkle::{EmptySubtreeRoots, SMT_DEPTH}; - - if account_ids.is_empty() { + if changed_account_ids.is_empty() { return Ok(()); } + self.update_storage_maps_in_forest(&changed_account_ids, block_num).await?; + + self.update_vaults_in_forest(&changed_account_ids, block_num).await?; + + Ok(()) + } + + /// Updates storage map SMTs in the forest for changed accounts + #[allow(clippy::too_many_lines)] + async fn update_storage_maps_in_forest( + &self, + changed_account_ids: &[AccountId], + block_num: BlockNumber, + ) -> Result<(), ApplyBlockError> { + use miden_objects::crypto::merkle::{EmptySubtreeRoots, SMT_DEPTH}; + tracing::debug!( target: COMPONENT, %block_num, - num_accounts = account_ids.len(), + num_accounts = changed_account_ids.len(), "Querying account storage from DB to populate SmtForest" ); // Query full storage for each updated account at this block let mut account_storages = Vec::new(); - for &account_id in &account_ids { - match self.db.select_account_storage_at_block(account_id, block_num).await { - Ok(storage) => { - account_storages.push((account_id, storage)); - }, - Err(e) => { - // Log error but don't fail the entire block application - // Forest will be missing this account but DB queries still work - tracing::warn!( - target: COMPONENT, - %account_id, - %block_num, - error = %e, - "Failed to query account storage for SmtForest update" - ); - }, - } + for &account_id in changed_account_ids { + let storage = self.db.select_account_storage_at_block(account_id, block_num).await?; + account_storages.push((account_id, storage)); } tracing::info!( @@ -540,7 +516,6 @@ impl State { for (account_id, storage) in &account_storages { // Iterate through each slot in the account storage for (slot_idx, slot) in storage.slots().iter().enumerate() { - let slot_idx_u8 = slot_idx as u8; // Only process Map-type slots if let miden_objects::account::StorageSlot::Map(storage_map) = slot { @@ -550,12 +525,12 @@ impl State { tracing::debug!( target: COMPONENT, %account_id, - slot_index = slot_idx_u8, + slot_index = slot_idx, num_entries = entries.len(), "Extracted Map slot entries" ); - map_slots_to_populate.push((*account_id, slot_idx_u8, entries)); + map_slots_to_populate.push((*account_id, slot_idx, entries)); } } } @@ -618,32 +593,9 @@ impl State { let mut new_roots = Vec::new(); for (account_id, slot_idx, prev_root, entries) in slots_with_prev_roots { - // Start with the previous root - let mut current_root = prev_root; - - // Insert all entries into the forest to build the new SMT - for (key, value) in entries { - match forest.insert(current_root, *key, *value) { - Ok(new_root) => { - current_root = new_root; - }, - Err(e) => { - // Log error but continue with other slots - tracing::error!( - target: COMPONENT, - %account_id, - slot_index = slot_idx, - error = ?e, - "Failed to insert entry into SmtForest" - ); - // Skip this slot by breaking out of entry loop - break; - }, - } - } - + let updated_root = forest.batch_insert(prev_root, entries.into_iter().cloned()).expect("Insertion into Forest always works"); // Store the final root after all insertions - new_roots.push((account_id, slot_idx, current_root)); + new_roots.push((account_id, slot_idx, updated_root)); tracing::debug!( target: COMPONENT, @@ -685,7 +637,17 @@ impl State { "Successfully tracked new roots (Step 5 complete)" ); - // VAULT TRACKING: Track vault SMT roots for structural sharing + Ok(()) + } + + /// Updates vault SMTs in the forest for changed accounts + async fn update_vaults_in_forest( + &self, + changed_account_ids: &[AccountId], + block_num: BlockNumber, + ) -> Result<(), ApplyBlockError> { + use miden_objects::crypto::merkle::{EmptySubtreeRoots, SMT_DEPTH}; + tracing::debug!( target: COMPONENT, %block_num, @@ -695,7 +657,7 @@ impl State { // Query vault assets for each updated account let mut vault_entries_to_populate = Vec::new(); - for &account_id in &account_ids { + for &account_id in changed_account_ids { match self.db.select_account_vault_at_block(account_id, block_num).await { Ok(entries) if !entries.is_empty() => { vault_entries_to_populate.push((account_id, entries)); @@ -748,26 +710,8 @@ impl State { let mut vault_new_roots = Vec::new(); for (account_id, prev_root, entries) in vaults_with_prev_roots { - let mut current_root = prev_root; - - for (key, value) in entries { - match forest.insert(current_root, key, value) { - Ok(new_root) => { - current_root = new_root; - }, - Err(e) => { - tracing::error!( - target: COMPONENT, - %account_id, - error = ?e, - "Failed to insert vault entry into SmtForest" - ); - break; - }, - } - } - - vault_new_roots.push((account_id, current_root)); + let updated_root = forest.batch_insert(prev_root, entries).expect("Database is consistent and always allows constructing a smt or forest"); + vault_new_roots.push((account_id, updated_root)); } drop(forest); @@ -1314,7 +1258,7 @@ impl State { return Err(DatabaseError::AccountNotPublic(account_id)); } - let (block_num, witness) = self.get_block_witness(block_num, account_id).await?; + let (block_num, witness) = self.get_account_witness(block_num, account_id).await?; let details = if let Some(request) = details { Some(self.fetch_requested_account_details(account_id, block_num, request).await?) @@ -1329,7 +1273,7 @@ impl State { /// /// If `block_num` is provided, returns the witness at that historical block, /// if not present, returns the witness at the latest block. - async fn get_block_witness( + async fn get_account_witness( &self, block_num: Option, account_id: AccountId, @@ -1375,48 +1319,65 @@ impl State { storage_requests, } = detail_request; + if !account_id.is_public() { + return + Err(DatabaseError::AccountNotPublic(account_id)); + } + + let forest_guard = self.storage_forest.read().await; + // First, get the account summary without deserializing the full account - let account_info = self.db.select_historical_account_at(account_id, block_num).await?; + // TODO we now still load details, but practically this should only return the summary + let AccountInfo { summary, details: _ } = self.db.select_historical_account_at(account_id, block_num).await?; + + // code + let account_code = if let Some(requested_commitment) = code_commitment { + if requested_commitment != summary.code_commitment { + // Client requested code and it doesn't match their cached version + // Query the code from the database + let code_bytes = self.db + .select_account_code_by_commitment(summary.code_commitment) + .await?; + + let code = miden_objects::account::AccountCode::read_from_bytes(&code_bytes)?; + Some(code) + } else { + // Client's cached code matches, no need to send it + None + } + } else { + // Client didn't request code + None + }; - // Ensure we have account details (only available for public accounts) - let Some(account) = account_info.details else { - return Err(DatabaseError::AccountNotPublic(account_id)); + // vault + let vault_details = match asset_vault_commitment { + Some(commitment) if commitment == summary.asset_vault_commitment => { + AccountVaultDetails::empty() + }, + Some(_) => AccountVaultDetails::new(account.vault()), + None => AccountVaultDetails::empty(), }; - let need_vault_from_account = asset_vault_commitment.is_some(); - let need_full_storage_maps_from_account = - storage_requests.iter().any(|req| matches!(req.slot_data, SlotData::All)); - let need_full_account = need_vault_from_account && need_full_storage_maps_from_account; + // storage requests + let storage_header: AccountStorageHeader = self.db.select_account_storage_header(account_id, block_num).await?; - if need_full_account { - // Inlined fetch_full_account_details - let storage_header = account.storage().to_header(); - let mut storage_map_details = - Vec::::with_capacity(storage_requests.len()); + let mut storage_map_details = + Vec::::with_capacity(storage_requests.len()); + + for StorageMapRequest { slot_index, slot_data } in storage_requests { + let Some(StorageSlot::Map(storage_map)) = + // FIXME TODO XXX load from SmtForest + account.storage().slots().get(slot_index as usize) + else { + return Err(AccountError::StorageSlotNotMap(slot_index).into()); + }; + let details = AccountStorageMapDetails::new(slot_index, slot_data, storage_map); + storage_map_details.push(details); + } - for StorageMapRequest { slot_index, slot_data } in storage_requests { - let Some(StorageSlot::Map(storage_map)) = - account.storage().slots().get(slot_index as usize) - else { - return Err(AccountError::StorageSlotNotMap(slot_index).into()); - }; - let details = AccountStorageMapDetails::new(slot_index, slot_data, storage_map); - storage_map_details.push(details); - } - // Only include account code if the commitment doesn't match - let account_code = code_commitment - .filter(|commitment| *commitment != account.code().commitment()) - .map(|_| account.code().to_bytes()); - // Handle vault details based on the provided commitment - let vault_details = match asset_vault_commitment { - Some(commitment) if commitment == account.vault().root() => { - AccountVaultDetails::empty() - }, - Some(_) => AccountVaultDetails::new(account.vault()), - None => AccountVaultDetails::empty(), - }; let account_header = AccountHeader::from(&account); @@ -1464,11 +1425,6 @@ impl State { let account_header = AccountHeader::from(&account); - // Only include account code if the commitment doesn't match - let account_code = code_commitment - .filter(|commitment| *commitment != account.code().commitment()) - .map(|_| account.code().to_bytes()); - Ok(AccountDetails { account_header, account_code, @@ -1481,138 +1437,6 @@ impl State { } } - /// Fetches full account details when full deserialization is required - fn fetch_full_account_details( - &self, - account: miden_objects::account::Account, - code_commitment: Option, - asset_vault_commitment: Option, - storage_requests: Vec, - ) -> Result { - let storage_header = account.storage().to_header(); - let storage_map_details = - self.process_storage_map_requests_full(&account, storage_requests)?; - - // Only include account code if the commitment doesn't match - let account_code = code_commitment - .filter(|commitment| *commitment != account.code().commitment()) - .map(|_| account.code().to_bytes()); - - // Handle vault details based on the provided commitment - let vault_details = match asset_vault_commitment { - Some(commitment) if commitment == account.vault().root() => { - AccountVaultDetails::empty() - }, - Some(_) => AccountVaultDetails::new(account.vault()), - None => AccountVaultDetails::empty(), - }; - - let account_header = AccountHeader::from(&account); - - Ok(AccountDetails { - account_header, - account_code, - vault_details, - storage_details: AccountStorageDetails { - header: storage_header, - map_details: storage_map_details, - }, - }) - } - - /// Fetches optimized account details by querying specific keys from DB - async fn fetch_optimized_account_details( - &self, - account: miden_objects::account::Account, - account_id: AccountId, - block_num: BlockNumber, - code_commitment: Option, - storage_requests: Vec, - ) -> Result { - let storage_header = account.storage().to_header(); - let storage_map_details = self - .process_storage_map_requests_optimized(account_id, block_num, storage_requests) - .await?; - let account_header = AccountHeader::from(&account); - - // Only include account code if the commitment doesn't match - let account_code = code_commitment - .filter(|commitment| *commitment != account.code().commitment()) - .map(|_| account.code().to_bytes()); - - Ok(AccountDetails { - account_header, - account_code, - vault_details: AccountVaultDetails::empty(), - storage_details: AccountStorageDetails { - header: storage_header, - map_details: storage_map_details, - }, - }) - } - - /// Processes storage map requests using full account data - fn process_storage_map_requests_full( - &self, - account: &miden_objects::account::Account, - storage_requests: Vec, - ) -> Result, DatabaseError> { - let mut storage_map_details = - Vec::::with_capacity(storage_requests.len()); - - for StorageMapRequest { slot_index, slot_data } in storage_requests { - let Some(StorageSlot::Map(storage_map)) = - account.storage().slots().get(slot_index as usize) - else { - return Err(AccountError::StorageSlotNotMap(slot_index).into()); - }; - let details = AccountStorageMapDetails::new(slot_index, slot_data, storage_map); - storage_map_details.push(details); - } - - Ok(storage_map_details) - } - - /// Processes storage map requests by querying DB for specific keys - async fn process_storage_map_requests_optimized( - &self, - account_id: AccountId, - block_num: BlockNumber, - storage_requests: Vec, - ) -> Result, DatabaseError> { - let mut storage_map_details = - Vec::::with_capacity(storage_requests.len()); - - for StorageMapRequest { slot_index, slot_data } in storage_requests { - let details = match slot_data { - SlotData::MapKeys(keys) => { - // Efficiently query specific keys from the DB - let map_entries = self - .db - .select_storage_map_keys_at_block( - account_id, - block_num, - slot_index, - keys.clone(), - ) - .await?; - - AccountStorageMapDetails::from_entries(slot_index, map_entries) - }, - SlotData::All => { - // This should not happen as we check for it in need_full_account - return Err(DatabaseError::DataCorrupted( - "SlotData::All should have been handled in need_full_account check" - .to_string(), - )); - }, - }; - storage_map_details.push(details); - } - - Ok(storage_map_details) - } - /// Returns storage map values for syncing within a block range. pub(crate) async fn get_storage_map_sync_values( &self, From 6a640771ff1eb3b51f3bfa46093bd468c14be65a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 2 Dec 2025 01:48:51 +0100 Subject: [PATCH 11/95] why --- crates/store/src/db/mod.rs | 7 +- .../store/src/db/models/queries/accounts.rs | 28 +++++- crates/store/src/state.rs | 93 ++++++++++++++----- 3 files changed, 98 insertions(+), 30 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 0b97ec5f24..186caceb15 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -13,7 +13,12 @@ use miden_objects::asset::{Asset, AssetVaultKey}; use miden_objects::block::{BlockHeader, BlockNoteIndex, BlockNumber, ProvenBlock}; use miden_objects::crypto::merkle::SparseMerklePath; use miden_objects::note::{ - NoteDetails, NoteId, NoteInclusionProof, NoteMetadata, NoteScript, Nullifier, + NoteDetails, + NoteId, + NoteInclusionProof, + NoteMetadata, + NoteScript, + Nullifier, }; use miden_objects::transaction::TransactionId; use tokio::sync::oneshot; diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 7e181530c1..25530d71ae 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -4,8 +4,16 @@ use diesel::prelude::{Queryable, QueryableByName}; use diesel::query_dsl::methods::SelectDsl; use diesel::sqlite::Sqlite; use diesel::{ - AsChangeset, BoolExpressionMethods, ExpressionMethods, Insertable, OptionalExtension, QueryDsl, - RunQueryDsl, Selectable, SelectableHelper, SqliteConnection, + AsChangeset, + BoolExpressionMethods, + ExpressionMethods, + Insertable, + OptionalExtension, + QueryDsl, + RunQueryDsl, + Selectable, + SelectableHelper, + SqliteConnection, }; use miden_lib::utils::{Deserializable, Serializable}; use miden_node_proto as proto; @@ -14,15 +22,25 @@ use miden_node_utils::limiter::{QueryParamAccountIdLimit, QueryParamLimiter}; use miden_objects::Word; use miden_objects::account::delta::AccountUpdateDetails; use miden_objects::account::{ - Account, AccountCode, AccountDelta, AccountId, AccountStorage, NonFungibleDeltaAction, - StorageSlot, StorageSlotType, + Account, + AccountCode, + AccountDelta, + AccountId, + AccountStorage, + NonFungibleDeltaAction, + StorageSlot, + StorageSlotType, }; use miden_objects::asset::{Asset, AssetVault, AssetVaultKey, FungibleAsset}; use miden_objects::block::{BlockAccountUpdate, BlockNumber}; use crate::constants::MAX_PAYLOAD_BYTES; use crate::db::models::conv::{ - SqlTypeConvert, nonce_to_raw_sql, raw_sql_to_nonce, raw_sql_to_slot, slot_to_raw_sql, + SqlTypeConvert, + nonce_to_raw_sql, + raw_sql_to_nonce, + raw_sql_to_slot, + slot_to_raw_sql, }; use crate::db::models::{serialize_vec, vec_raw_try_into}; use crate::db::{AccountVaultValue, schema}; diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 7037dc6eb4..af0fb2367b 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -9,9 +9,16 @@ use std::path::Path; use std::sync::Arc; use miden_node_proto::domain::account::{ - AccountDetailRequest, AccountDetails, AccountInfo, AccountProofRequest, AccountProofResponse, - AccountStorageDetails, AccountStorageMapDetails, AccountVaultDetails, NetworkAccountPrefix, - SlotData, StorageMapRequest, + AccountDetailRequest, + AccountDetails, + AccountInfo, + AccountProofRequest, + AccountProofResponse, + AccountStorageDetails, + AccountStorageMapDetails, + AccountVaultDetails, + NetworkAccountPrefix, + StorageMapRequest, }; use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; @@ -20,12 +27,26 @@ use miden_objects::account::{AccountHeader, AccountId, StorageSlot}; use miden_objects::block::account_tree::{AccountTree, account_id_to_smt_key}; use miden_objects::block::nullifier_tree::NullifierTree; use miden_objects::block::{ - AccountWitness, BlockHeader, BlockInputs, BlockNumber, Blockchain, NullifierWitness, + AccountWitness, + BlockHeader, + BlockInputs, + BlockNumber, + Blockchain, + NullifierWitness, ProvenBlock, }; use miden_objects::crypto::merkle::{ - Forest, LargeSmt, MemoryStorage, Mmr, MmrDelta, MmrPeaks, MmrProof, PartialMmr, SmtForest, - SmtProof, SmtStorage, + Forest, + LargeSmt, + MemoryStorage, + Mmr, + MmrDelta, + MmrPeaks, + MmrProof, + PartialMmr, + SmtForest, + SmtProof, + SmtStorage, }; use miden_objects::note::{NoteDetails, NoteId, NoteScript, Nullifier}; use miden_objects::transaction::{OutputNote, PartialBlockchain}; @@ -36,13 +57,25 @@ use tracing::{info, info_span, instrument}; use crate::blocks::BlockStore; use crate::db::models::Page; -use crate::db::models::queries::{StorageMapValuesPage, select_account_storage_headers_at_block}; +use crate::db::models::queries::StorageMapValuesPage; use crate::db::{ - AccountVaultValue, Db, NoteRecord, NoteSyncUpdate, NullifierInfo, StateSyncUpdate, + AccountVaultValue, + Db, + NoteRecord, + NoteSyncUpdate, + NullifierInfo, + StateSyncUpdate, }; use crate::errors::{ - ApplyBlockError, DatabaseError, GetBatchInputsError, GetBlockHeaderError, GetBlockInputsError, - GetCurrentBlockchainDataError, InvalidBlockError, NoteSyncError, StateInitializationError, + ApplyBlockError, + DatabaseError, + GetBatchInputsError, + GetBlockHeaderError, + GetBlockInputsError, + GetCurrentBlockchainDataError, + InvalidBlockError, + NoteSyncError, + StateInitializationError, StateSyncError, }; use crate::{AccountTreeWithHistory, COMPONENT, DataDirectory}; @@ -162,10 +195,19 @@ impl State { }; // load all accounts from the table - // TODO: make `select_all_account_at(block_num)` to be precise; if ACID is upheld, it's not necessary in theory - let acc_account_ids = db.select_all_account_commitments().await?; - let acc_account_ids = Vec::from_iter(acc_account_ids.into_iter().map(|(account_id, _)| acc_account_ids)); - me.update_storage_forest_from_db(acc_account_ids, latest_block_num)?; + // TODO: make `select_all_account_at(block_num)` to be precise; if ACID is upheld, it's not + // necessary in theory + let acc_account_ids = me.db.select_all_account_commitments().await?; + let acc_account_ids = + acc_account_ids.into_iter().map(|(account_id, _)| account_id).collect(); + me.update_storage_forest_from_db(acc_account_ids, latest_block_num) + .await + .map_err(|e| { + StateInitializationError::DatabaseError(DatabaseError::InteractError(format!( + "Failed to update storage forest: {}", + e + ))) + })?; Ok(me) } @@ -516,7 +558,6 @@ impl State { for (account_id, storage) in &account_storages { // Iterate through each slot in the account storage for (slot_idx, slot) in storage.slots().iter().enumerate() { - // Only process Map-type slots if let miden_objects::account::StorageSlot::Map(storage_map) = slot { // Extract all (key, value) entries from this StorageMap @@ -593,7 +634,9 @@ impl State { let mut new_roots = Vec::new(); for (account_id, slot_idx, prev_root, entries) in slots_with_prev_roots { - let updated_root = forest.batch_insert(prev_root, entries.into_iter().cloned()).expect("Insertion into Forest always works"); + let updated_root = forest + .batch_insert(prev_root, entries.into_iter().map(|(k, v)| (*k, *v))) + .expect("Insertion into Forest always works"); // Store the final root after all insertions new_roots.push((account_id, slot_idx, updated_root)); @@ -710,7 +753,9 @@ impl State { let mut vault_new_roots = Vec::new(); for (account_id, prev_root, entries) in vaults_with_prev_roots { - let updated_root = forest.batch_insert(prev_root, entries).expect("Database is consistent and always allows constructing a smt or forest"); + let updated_root = forest + .batch_insert(prev_root, entries) + .expect("Database is consistent and always allows constructing a smt or forest"); vault_new_roots.push((account_id, updated_root)); } drop(forest); @@ -1320,8 +1365,7 @@ impl State { } = detail_request; if !account_id.is_public() { - return - Err(DatabaseError::AccountNotPublic(account_id)); + return Err(DatabaseError::AccountNotPublic(account_id)); } let forest_guard = self.storage_forest.read().await; @@ -1346,7 +1390,6 @@ impl State { None } } else { - // Client didn't request code None }; @@ -1366,12 +1409,14 @@ impl State { Vec::::with_capacity(storage_requests.len()); for StorageMapRequest { slot_index, slot_data } in storage_requests { - let Some(StorageSlot::Map(storage_map)) = - // FIXME TODO XXX load from SmtForest - account.storage().slots().get(slot_index as usize) - else { + let Some(slot) = account.storage().slots().get(slot_index as usize) else { + continue; + }; + + let StorageSlot::Map(storage_map) = slot else { return Err(AccountError::StorageSlotNotMap(slot_index).into()); }; + let details = AccountStorageMapDetails::new(slot_index, slot_data, storage_map); storage_map_details.push(details); } From 9d5806e3013445147bdd96d56ca992f8a68463b4 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 2 Dec 2025 10:35:23 +0100 Subject: [PATCH 12/95] y --- crates/proto/src/domain/account.rs | 58 +++--- crates/store/src/db/mod.rs | 28 +++ .../store/src/db/models/queries/accounts.rs | 165 ++++++++++++++++ crates/store/src/db/tests.rs | 52 ++++++ crates/store/src/state.rs | 176 +++++------------- 5 files changed, 318 insertions(+), 161 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 1e02187865..df41ecde87 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -3,12 +3,7 @@ use std::fmt::{Debug, Display, Formatter}; use miden_node_utils::formatting::format_opt; use miden_objects::Word; use miden_objects::account::{ - Account, - AccountHeader, - AccountId, - AccountStorageHeader, - StorageMap, - StorageSlotType, + Account, AccountHeader, AccountId, AccountStorageHeader, StorageMap, StorageSlotType, }; use miden_objects::asset::{Asset, AssetVault}; use miden_objects::block::{AccountWitness, BlockNumber}; @@ -377,6 +372,27 @@ impl AccountVaultDetails { } } + /// Creates `AccountVaultDetails` from vault entries (key-value pairs). + /// + /// This is useful when entries have been fetched directly from the database + /// rather than extracted from an AssetVault. + /// + /// The entries are `(vault_key, asset)` pairs where `asset` is a Word representation. + pub fn from_entries(entries: Vec<(Word, Word)>) -> Result { + let too_many_assets = entries.len() > Self::MAX_RETURN_ENTRIES; + + if too_many_assets { + return Ok(Self::too_many()); + } + + let assets = entries + .into_iter() + .map(|(_key, asset_word)| Asset::try_from(asset_word)) + .collect::, _>>()?; + + Ok(Self { too_many_assets: false, assets }) + } + fn too_many() -> Self { Self { too_many_assets: true, @@ -420,6 +436,10 @@ impl From for proto::rpc_store::AccountVaultDetails { pub struct AccountStorageMapDetails { pub slot_index: u8, pub too_many_entries: bool, + // TODO the following is only for the case when _all_ entries are included + // TODO for partials, we also need to provide merkle proofs / a partial SMT with inner nodes + // Reason: if all leaf values are included, one can reconstruct the entire SMT, if just one + // is missing one cannot pub map_entries: Vec<(Word, Word)>, } @@ -429,7 +449,7 @@ impl AccountStorageMapDetails { pub fn new(slot_index: u8, slot_data: SlotData, storage_map: &StorageMap) -> Self { match slot_data { SlotData::All => Self::from_all_entries(slot_index, storage_map), - SlotData::MapKeys(keys) => Self::from_specific_keys(slot_index, &keys[..], storage_map), + SlotData::MapKeys(keys) => Self::from_all_entries(slot_index, storage_map), // TODO use from_specific_keys } } @@ -446,31 +466,13 @@ impl AccountStorageMapDetails { } } + // TODO this is + #[allow(dead_code)] fn from_specific_keys(slot_index: u8, keys: &[Word], storage_map: &StorageMap) -> Self { if keys.len() > Self::MAX_RETURN_ENTRIES { Self::too_many_entries(slot_index) } else { - // Query specific keys from the storage map. - // StorageMap::get returns the value for a given key, or EMPTY_WORD if not present. - // We only return entries that actually exist in the map (non-empty values). - let map_entries: Vec<(Word, Word)> = keys - .iter() - .filter_map(|key| { - let value = storage_map.get(key); - // Only include entries with non-empty values - if value == miden_objects::EMPTY_WORD { - None - } else { - Some((*key, value)) - } - }) - .collect(); - - Self { - slot_index, - too_many_entries: false, - map_entries, - } + todo!("construct a partial SMT / set of key values") } } diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 186caceb15..f785b766c3 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -477,6 +477,34 @@ impl Db { .await } + /// Queries the account code for a specific account at a specific block number. + /// + /// Returns `None` if the account doesn't exist at that block or has no code. + pub async fn select_account_code_at_block( + &self, + account_id: AccountId, + block_num: BlockNumber, + ) -> Result>> { + self.transact("Get account code at block", move |conn| { + queries::select_account_code_at_block(conn, account_id, block_num) + }) + .await + } + + /// Queries the account header for a specific account at a specific block number. + /// + /// Returns `None` if the account doesn't exist at that block. + pub async fn select_account_header_at_block( + &self, + account_id: AccountId, + block_num: BlockNumber, + ) -> Result> { + self.transact("Get account header at block", move |conn| { + queries::select_account_header_at_block(conn, account_id, block_num) + }) + .await + } + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn get_state_sync( &self, diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 25530d71ae..401e607bce 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -20,11 +20,13 @@ use miden_node_proto as proto; use miden_node_proto::domain::account::{AccountInfo, AccountSummary}; use miden_node_utils::limiter::{QueryParamAccountIdLimit, QueryParamLimiter}; use miden_objects::Word; +use miden_objects::{Felt, FieldElement}; use miden_objects::account::delta::AccountUpdateDetails; use miden_objects::account::{ Account, AccountCode, AccountDelta, + AccountHeader, AccountId, AccountStorage, NonFungibleDeltaAction, @@ -1381,3 +1383,166 @@ pub(crate) fn select_account_vault_at_block( Ok(entries) } + +/// Computes the storage commitment from a list of slot commitments. +/// +/// This replicates the logic from `AccountStorage::commitment()` which hashes all slot +/// commitments together. +/// +/// # Arguments +/// +/// * `slot_commitments` - Vector of slot commitment words +/// +/// # Returns +/// +/// The storage commitment as a `Word` +fn compute_storage_commitment(slot_commitments: &[Word]) -> Word { + use miden_objects::crypto::hash::rpo::Rpo256; + + let elements: Vec = slot_commitments + .iter() + .flat_map(|w| w.iter()) + .copied() + .collect(); + + Rpo256::hash_elements(&elements).into() +} + +/// Queries the account code for a specific account at a specific block number. +/// +/// Returns `None` if: +/// - The account doesn't exist at that block +/// - The account has no code (private account or account without code commitment) +/// +/// # Arguments +/// +/// * `conn` - Database connection +/// * `account_id` - The account ID to query +/// * `block_num` - The block number at which to query the account code +/// +/// # Returns +/// +/// * `Ok(Some(Vec))` - The account code bytes if found +/// * `Ok(None)` - If account doesn't exist or has no code +/// * `Err(DatabaseError)` - If there's a database error +pub(crate) fn select_account_code_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result>, DatabaseError> { + use schema::{account_codes, accounts}; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = i64::from(block_num.as_u32()); + + // Query the accounts table to get the code_commitment at the specified block + // Then join with account_codes to get the actual code + let result: Option> = SelectDsl::select( + accounts::table + .inner_join(account_codes::table) + .filter(accounts::account_id.eq(&account_id_bytes)) + .filter(accounts::block_num.eq(block_num_sql)), + account_codes::code, + ) + .first(conn) + .optional()?; + + Ok(result) +} + +/// Queries the account header for a specific account at a specific block number. +/// +/// This reconstructs the AccountHeader by joining multiple tables: +/// - `accounts` table for account_id, nonce, `code_commitment` +/// - `account_vault_headers` table for `vault_root` +/// - `account_storage_headers` table for storage slot commitments (to compute `storage_commitment`) +/// +/// Returns `None` if the account doesn't exist at that block. +/// +/// # Arguments +/// +/// * `conn` - Database connection +/// * `account_id` - The account ID to query +/// * `block_num` - The block number at which to query the account header +/// +/// # Returns +/// +/// * `Ok(Some(AccountHeader))` - The account header if found +/// * `Ok(None)` - If account doesn't exist at that block +/// * `Err(DatabaseError)` - If there's a database error +pub(crate) fn select_account_header_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { + use schema::{account_storage_headers, account_vault_headers, accounts}; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + + let account_data: Option<(Option>, Option)> = SelectDsl::select( + accounts::table + .filter(accounts::account_id.eq(&account_id_bytes)) + .filter(accounts::block_num.eq(block_num_sql)), + (accounts::code_commitment, accounts::nonce), + ) + .first(conn) + .optional()?; + + let Some((code_commitment_bytes, nonce_raw)) = account_data else { + return Ok(None); + }; + + let vault_root_bytes: Option> = SelectDsl::select( + account_vault_headers::table + .filter(account_vault_headers::account_id.eq(&account_id_bytes)) + .filter(account_vault_headers::block_num.eq(block_num_sql)), + account_vault_headers::vault_root, + ) + .first(conn) + .optional()?; + + let storage_slots: Vec<(i32, i32, Vec)> = SelectDsl::select( + account_storage_headers::table + .filter(account_storage_headers::account_id.eq(&account_id_bytes)) + .filter(account_storage_headers::block_num.eq(block_num_sql)) + .order(account_storage_headers::slot_index.asc()), + ( + account_storage_headers::slot_index, + account_storage_headers::slot_type, + account_storage_headers::slot_commitment, + ), + ) + .load(conn)?; + + let slot_commitments: Vec = storage_slots + .into_iter() + .map(|(_slot_index, _slot_type, commitment_bytes)| { + Word::read_from_bytes(&commitment_bytes) + }) + .collect::, _>>()?; + + let storage_commitment = compute_storage_commitment(&slot_commitments); + + let code_commitment = code_commitment_bytes + .map(|bytes| Word::read_from_bytes(&bytes)) + .transpose()? + .unwrap_or(Word::default()); + + let nonce = nonce_raw + .map(raw_sql_to_nonce) + .unwrap_or(Felt::ZERO); + + let vault_root = vault_root_bytes + .map(|bytes| Word::read_from_bytes(&bytes)) + .transpose()? + .unwrap_or(Word::default()); + + Ok(Some(AccountHeader::new( + account_id, + nonce, + vault_root, + storage_commitment, + code_commitment, + ))) +} diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index f00822424a..ba6da1bb3b 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -8,6 +8,7 @@ use diesel::{Connection, SqliteConnection}; use miden_lib::account::auth::AuthRpoFalcon512; use miden_lib::note::create_p2id_note; use miden_lib::transaction::TransactionKernel; +use miden_lib::utils::Serializable; use miden_node_proto::domain::account::AccountSummary; use miden_node_utils::fee::test_fee_params; use miden_objects::account::auth::PublicKeyCommitment; @@ -1896,3 +1897,54 @@ fn test_storage_header_is_latest_flag() { panic!("Expected Value slot with value_1"); } } + +#[test] +fn test_select_account_code_at_block() { + let mut conn = create_db(); + + let block_num_1 = BlockNumber::from(1); + let block_num_2 = BlockNumber::from(2); + + // Create an account with code at block 1 using the existing mock function + let account = mock_account_code_and_storage( + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + [], + None, + ); + + // Use the actual account ID from the created account + let account_id = account.id(); + + // Get the code bytes before inserting + let expected_code = account.code().to_bytes(); + + // Insert the account at block 1 + queries::upsert_accounts( + &mut conn, + &[BlockAccountUpdate::new( + account_id, + account.commitment(), + AccountUpdateDetails::Delta(AccountDelta::try_from(account).unwrap()), + )], + block_num_1, + ) + .unwrap(); + + // Query code at block 1 - should return the code + let code_at_1 = queries::select_account_code_at_block(&mut conn, account_id, block_num_1) + .unwrap() + .expect("Code should exist at block 1"); + assert_eq!(code_at_1, expected_code); + + // Query code at non-existent block - should return None + let code_at_2 = queries::select_account_code_at_block(&mut conn, account_id, block_num_2) + .unwrap(); + assert!(code_at_2.is_none(), "Code should not exist at block 2"); + + // Query code for non-existent account - should return None + let other_account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + let code_other = queries::select_account_code_at_block(&mut conn, other_account_id, block_num_1) + .unwrap(); + assert!(code_other.is_none(), "Code should not exist for non-existent account"); +} diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index af0fb2367b..a0902dae64 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -9,44 +9,23 @@ use std::path::Path; use std::sync::Arc; use miden_node_proto::domain::account::{ - AccountDetailRequest, - AccountDetails, - AccountInfo, - AccountProofRequest, - AccountProofResponse, - AccountStorageDetails, - AccountStorageMapDetails, - AccountVaultDetails, - NetworkAccountPrefix, + AccountDetailRequest, AccountDetails, AccountInfo, AccountProofRequest, AccountProofResponse, + AccountStorageDetails, AccountStorageMapDetails, AccountVaultDetails, NetworkAccountPrefix, StorageMapRequest, }; use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; -use miden_objects::account::{AccountHeader, AccountId, StorageSlot}; +use miden_objects::account::{AccountId, StorageSlot}; use miden_objects::block::account_tree::{AccountTree, account_id_to_smt_key}; use miden_objects::block::nullifier_tree::NullifierTree; use miden_objects::block::{ - AccountWitness, - BlockHeader, - BlockInputs, - BlockNumber, - Blockchain, - NullifierWitness, + AccountWitness, BlockHeader, BlockInputs, BlockNumber, Blockchain, NullifierWitness, ProvenBlock, }; use miden_objects::crypto::merkle::{ - Forest, - LargeSmt, - MemoryStorage, - Mmr, - MmrDelta, - MmrPeaks, - MmrProof, - PartialMmr, - SmtForest, - SmtProof, - SmtStorage, + Forest, LargeSmt, MemoryStorage, Mmr, MmrDelta, MmrPeaks, MmrProof, PartialMmr, SmtForest, + SmtProof, SmtStorage, }; use miden_objects::note::{NoteDetails, NoteId, NoteScript, Nullifier}; use miden_objects::transaction::{OutputNote, PartialBlockchain}; @@ -59,23 +38,11 @@ use crate::blocks::BlockStore; use crate::db::models::Page; use crate::db::models::queries::StorageMapValuesPage; use crate::db::{ - AccountVaultValue, - Db, - NoteRecord, - NoteSyncUpdate, - NullifierInfo, - StateSyncUpdate, + AccountVaultValue, Db, NoteRecord, NoteSyncUpdate, NullifierInfo, StateSyncUpdate, }; use crate::errors::{ - ApplyBlockError, - DatabaseError, - GetBatchInputsError, - GetBlockHeaderError, - GetBlockInputsError, - GetCurrentBlockchainDataError, - InvalidBlockError, - NoteSyncError, - StateInitializationError, + ApplyBlockError, DatabaseError, GetBatchInputsError, GetBlockHeaderError, GetBlockInputsError, + GetCurrentBlockchainDataError, InvalidBlockError, NoteSyncError, StateInitializationError, StateSyncError, }; use crate::{AccountTreeWithHistory, COMPONENT, DataDirectory}; @@ -571,7 +538,7 @@ impl State { "Extracted Map slot entries" ); - map_slots_to_populate.push((*account_id, slot_idx, entries)); + map_slots_to_populate.push((*account_id, slot_idx as u8, entries)); } } } @@ -1368,52 +1335,45 @@ impl State { return Err(DatabaseError::AccountNotPublic(account_id)); } - let forest_guard = self.storage_forest.read().await; - - // First, get the account summary without deserializing the full account - // TODO we now still load details, but practically this should only return the summary - let AccountInfo { summary, details: _ } = self.db.select_historical_account_at(account_id, block_num).await?; - - // code - let account_code = if let Some(requested_commitment) = code_commitment { - if requested_commitment != summary.code_commitment { - // Client requested code and it doesn't match their cached version - // Query the code from the database - let code_bytes = self.db - .select_account_code_by_commitment(summary.code_commitment) - .await?; - - let code = miden_objects::account::AccountCode::read_from_bytes(&code_bytes)?; - Some(code) - } else { - // Client's cached code matches, no need to send it - None - } - } else { - None + let account_header = self + .db + .select_account_header_at_block(account_id, block_num) + .await? + .ok_or_else(|| DatabaseError::AccountNotPublic(account_id))?; + + let account_code = match code_commitment { + Some(commitment) if commitment == account_header.code_commitment() => None, + Some(_) => self.db.select_account_code_at_block(account_id, block_num).await?, + None => None, }; - // vault let vault_details = match asset_vault_commitment { - Some(commitment) if commitment == summary.asset_vault_commitment => { + Some(commitment) if commitment == account_header.vault_root() => { AccountVaultDetails::empty() }, - Some(_) => AccountVaultDetails::new(account.vault()), - None => AccountVaultDetails::empty(), + Some(_) | None if asset_vault_commitment.is_some() => { + let vault_entries = + self.db.select_account_vault_at_block(account_id, block_num).await?; + AccountVaultDetails::from_entries(vault_entries).map_err(|e| { + DatabaseError::InteractError(format!("Failed to parse vault assets: {e}")) + })? + }, + _ => AccountVaultDetails::empty(), }; - // storage requests - let storage_header: AccountStorageHeader = self.db.select_account_storage_header(account_id, block_num).await?; - + // TODO: don't load the entire store at once, load what is required + let store = self.db.select_account_storage_at_block(account_id, block_num).await?; + let storage_header = store.to_header(); let mut storage_map_details = Vec::::with_capacity(storage_requests.len()); for StorageMapRequest { slot_index, slot_data } in storage_requests { - let Some(slot) = account.storage().slots().get(slot_index as usize) else { + let Some(slot) = store.slots().get(slot_index as usize) else { continue; }; let StorageSlot::Map(storage_map) = slot else { + // TODO: what to do with value entries? Is it ok to ignore them? return Err(AccountError::StorageSlotNotMap(slot_index).into()); }; @@ -1421,65 +1381,15 @@ impl State { storage_map_details.push(details); } - - - - let account_header = AccountHeader::from(&account); - - Ok(AccountDetails { - account_header, - account_code, - vault_details, - storage_details: AccountStorageDetails { - header: storage_header, - map_details: storage_map_details, - }, - }) - } else { - // Inlined fetch_optimized_account_details - let storage_header = account.storage().to_header(); - let mut storage_map_details = - Vec::::with_capacity(storage_requests.len()); - - for StorageMapRequest { slot_index, slot_data } in storage_requests { - let details = match slot_data { - SlotData::MapKeys(keys) => { - // Efficiently query specific keys from the DB - let map_entries = self - .db - .select_storage_map_keys_at_block( - account_id, - block_num, - slot_index, - keys.clone(), - ) - .await?; - - AccountStorageMapDetails::from_entries(slot_index, map_entries) - }, - SlotData::All => { - // This should not happen as we check for it in need_full_account - return Err(DatabaseError::DataCorrupted( - "SlotData::All should have been handled in need_full_account check" - .to_string(), - )); - }, - }; - storage_map_details.push(details); - } - - let account_header = AccountHeader::from(&account); - - Ok(AccountDetails { - account_header, - account_code, - vault_details: AccountVaultDetails::empty(), - storage_details: AccountStorageDetails { - header: storage_header, - map_details: storage_map_details, - }, - }) - } + Ok(AccountDetails { + account_header, + account_code, + vault_details, + storage_details: AccountStorageDetails { + header: storage_header, + map_details: storage_map_details, + }, + }) } /// Returns storage map values for syncing within a block range. From 2964a93fd5571cea8c15e274078162d0922af951 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 2 Dec 2025 16:54:12 +0100 Subject: [PATCH 13/95] y --- crates/proto/src/domain/account.rs | 17 ++++--- .../store/src/db/models/queries/accounts.rs | 21 +++----- crates/store/src/db/tests.rs | 12 ++--- crates/store/src/state.rs | 49 ++++++++++++++++--- 4 files changed, 64 insertions(+), 35 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index df41ecde87..a70022ea14 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -3,7 +3,12 @@ use std::fmt::{Debug, Display, Formatter}; use miden_node_utils::formatting::format_opt; use miden_objects::Word; use miden_objects::account::{ - Account, AccountHeader, AccountId, AccountStorageHeader, StorageMap, StorageSlotType, + Account, + AccountHeader, + AccountId, + AccountStorageHeader, + StorageMap, + StorageSlotType, }; use miden_objects::asset::{Asset, AssetVault}; use miden_objects::block::{AccountWitness, BlockNumber}; @@ -375,7 +380,7 @@ impl AccountVaultDetails { /// Creates `AccountVaultDetails` from vault entries (key-value pairs). /// /// This is useful when entries have been fetched directly from the database - /// rather than extracted from an AssetVault. + /// rather than extracted from an `AssetVault`. /// /// The entries are `(vault_key, asset)` pairs where `asset` is a Word representation. pub fn from_entries(entries: Vec<(Word, Word)>) -> Result { @@ -449,7 +454,7 @@ impl AccountStorageMapDetails { pub fn new(slot_index: u8, slot_data: SlotData, storage_map: &StorageMap) -> Self { match slot_data { SlotData::All => Self::from_all_entries(slot_index, storage_map), - SlotData::MapKeys(keys) => Self::from_all_entries(slot_index, storage_map), // TODO use from_specific_keys + SlotData::MapKeys(_keys) => Self::from_all_entries(slot_index, storage_map), /* TODO use from_specific_keys */ } } @@ -468,7 +473,7 @@ impl AccountStorageMapDetails { // TODO this is #[allow(dead_code)] - fn from_specific_keys(slot_index: u8, keys: &[Word], storage_map: &StorageMap) -> Self { + fn from_specific_keys(slot_index: u8, keys: &[Word], _storage_map: &StorageMap) -> Self { if keys.len() > Self::MAX_RETURN_ENTRIES { Self::too_many_entries(slot_index) } else { @@ -476,9 +481,9 @@ impl AccountStorageMapDetails { } } - /// Creates an AccountStorageMapDetails from already-queried entries (e.g., from database). + /// Creates an `AccountStorageMapDetails` from already-queried entries (e.g., from database). /// This is useful when entries have been fetched directly rather than extracted from a - /// StorageMap. + /// `StorageMap`. pub fn from_entries(slot_index: u8, map_entries: Vec<(Word, Word)>) -> Self { let too_many_entries = map_entries.len() > Self::MAX_RETURN_ENTRIES; let map_entries = if too_many_entries { Vec::new() } else { map_entries }; diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 401e607bce..14f9263bda 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -19,8 +19,6 @@ use miden_lib::utils::{Deserializable, Serializable}; use miden_node_proto as proto; use miden_node_proto::domain::account::{AccountInfo, AccountSummary}; use miden_node_utils::limiter::{QueryParamAccountIdLimit, QueryParamLimiter}; -use miden_objects::Word; -use miden_objects::{Felt, FieldElement}; use miden_objects::account::delta::AccountUpdateDetails; use miden_objects::account::{ Account, @@ -35,6 +33,7 @@ use miden_objects::account::{ }; use miden_objects::asset::{Asset, AssetVault, AssetVaultKey, FungibleAsset}; use miden_objects::block::{BlockAccountUpdate, BlockNumber}; +use miden_objects::{Felt, FieldElement, Word}; use crate::constants::MAX_PAYLOAD_BYTES; use crate::db::models::conv::{ @@ -1398,13 +1397,9 @@ pub(crate) fn select_account_vault_at_block( /// The storage commitment as a `Word` fn compute_storage_commitment(slot_commitments: &[Word]) -> Word { use miden_objects::crypto::hash::rpo::Rpo256; - - let elements: Vec = slot_commitments - .iter() - .flat_map(|w| w.iter()) - .copied() - .collect(); - + + let elements: Vec = slot_commitments.iter().flat_map(|w| w.iter()).copied().collect(); + Rpo256::hash_elements(&elements).into() } @@ -1517,9 +1512,7 @@ pub(crate) fn select_account_header_at_block( let slot_commitments: Vec = storage_slots .into_iter() - .map(|(_slot_index, _slot_type, commitment_bytes)| { - Word::read_from_bytes(&commitment_bytes) - }) + .map(|(_slot_index, _slot_type, commitment_bytes)| Word::read_from_bytes(&commitment_bytes)) .collect::, _>>()?; let storage_commitment = compute_storage_commitment(&slot_commitments); @@ -1529,9 +1522,7 @@ pub(crate) fn select_account_header_at_block( .transpose()? .unwrap_or(Word::default()); - let nonce = nonce_raw - .map(raw_sql_to_nonce) - .unwrap_or(Felt::ZERO); + let nonce = nonce_raw.map_or(Felt::ZERO, raw_sql_to_nonce); let vault_root = vault_root_bytes .map(|bytes| Word::read_from_bytes(&bytes)) diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index ba6da1bb3b..5fa760bae7 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1912,10 +1912,10 @@ fn test_select_account_code_at_block() { [], None, ); - + // Use the actual account ID from the created account let account_id = account.id(); - + // Get the code bytes before inserting let expected_code = account.code().to_bytes(); @@ -1938,13 +1938,13 @@ fn test_select_account_code_at_block() { assert_eq!(code_at_1, expected_code); // Query code at non-existent block - should return None - let code_at_2 = queries::select_account_code_at_block(&mut conn, account_id, block_num_2) - .unwrap(); + let code_at_2 = + queries::select_account_code_at_block(&mut conn, account_id, block_num_2).unwrap(); assert!(code_at_2.is_none(), "Code should not exist at block 2"); // Query code for non-existent account - should return None let other_account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - let code_other = queries::select_account_code_at_block(&mut conn, other_account_id, block_num_1) - .unwrap(); + let code_other = + queries::select_account_code_at_block(&mut conn, other_account_id, block_num_1).unwrap(); assert!(code_other.is_none(), "Code should not exist for non-existent account"); } diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index a0902dae64..fdcef630aa 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -9,8 +9,15 @@ use std::path::Path; use std::sync::Arc; use miden_node_proto::domain::account::{ - AccountDetailRequest, AccountDetails, AccountInfo, AccountProofRequest, AccountProofResponse, - AccountStorageDetails, AccountStorageMapDetails, AccountVaultDetails, NetworkAccountPrefix, + AccountDetailRequest, + AccountDetails, + AccountInfo, + AccountProofRequest, + AccountProofResponse, + AccountStorageDetails, + AccountStorageMapDetails, + AccountVaultDetails, + NetworkAccountPrefix, StorageMapRequest, }; use miden_node_proto::domain::batch::BatchInputs; @@ -20,12 +27,26 @@ use miden_objects::account::{AccountId, StorageSlot}; use miden_objects::block::account_tree::{AccountTree, account_id_to_smt_key}; use miden_objects::block::nullifier_tree::NullifierTree; use miden_objects::block::{ - AccountWitness, BlockHeader, BlockInputs, BlockNumber, Blockchain, NullifierWitness, + AccountWitness, + BlockHeader, + BlockInputs, + BlockNumber, + Blockchain, + NullifierWitness, ProvenBlock, }; use miden_objects::crypto::merkle::{ - Forest, LargeSmt, MemoryStorage, Mmr, MmrDelta, MmrPeaks, MmrProof, PartialMmr, SmtForest, - SmtProof, SmtStorage, + Forest, + LargeSmt, + MemoryStorage, + Mmr, + MmrDelta, + MmrPeaks, + MmrProof, + PartialMmr, + SmtForest, + SmtProof, + SmtStorage, }; use miden_objects::note::{NoteDetails, NoteId, NoteScript, Nullifier}; use miden_objects::transaction::{OutputNote, PartialBlockchain}; @@ -38,11 +59,23 @@ use crate::blocks::BlockStore; use crate::db::models::Page; use crate::db::models::queries::StorageMapValuesPage; use crate::db::{ - AccountVaultValue, Db, NoteRecord, NoteSyncUpdate, NullifierInfo, StateSyncUpdate, + AccountVaultValue, + Db, + NoteRecord, + NoteSyncUpdate, + NullifierInfo, + StateSyncUpdate, }; use crate::errors::{ - ApplyBlockError, DatabaseError, GetBatchInputsError, GetBlockHeaderError, GetBlockInputsError, - GetCurrentBlockchainDataError, InvalidBlockError, NoteSyncError, StateInitializationError, + ApplyBlockError, + DatabaseError, + GetBatchInputsError, + GetBlockHeaderError, + GetBlockInputsError, + GetCurrentBlockchainDataError, + InvalidBlockError, + NoteSyncError, + StateInitializationError, StateSyncError, }; use crate::{AccountTreeWithHistory, COMPONENT, DataDirectory}; From 9416a637add6c9b46acd4a205b85a4f5860eeaa2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 2 Dec 2025 17:52:05 +0100 Subject: [PATCH 14/95] review comments --- crates/store/src/db/mod.rs | 33 +---- .../store/src/db/models/queries/accounts.rs | 138 +----------------- crates/store/src/db/tests.rs | 52 ------- 3 files changed, 4 insertions(+), 219 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index f785b766c3..653505ebec 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -409,18 +409,7 @@ impl Db { .await } - /// Loads account details at a specific block number from the DB. - #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub async fn select_historical_account_at( - &self, - id: AccountId, - block_num: BlockNumber, - ) -> Result { - self.transact("Get historical account details", move |conn| { - queries::select_historical_account_at(conn, id, block_num) - }) - .await - } + /// Loads public account details from the DB based on the account ID's prefix. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] @@ -618,26 +607,6 @@ impl Db { .await } - /// Selects specific storage map keys at a specific block from the DB - /// - /// This method is optimized for querying specific keys without deserializing the entire - /// account, which is much faster for historical queries. - #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub async fn select_storage_map_keys_at_block( - &self, - account_id: AccountId, - block_num: BlockNumber, - slot_index: u8, - keys: Vec, - ) -> Result> { - self.transact("select storage map keys at block", move |conn| { - models::queries::select_storage_map_keys_at_block( - conn, account_id, block_num, slot_index, &keys, - ) - }) - .await - } - /// Runs database optimization. #[instrument(level = "debug", target = COMPONENT, skip_all, err)] pub async fn optimize(&self) -> Result<(), DatabaseError> { diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 14f9263bda..3c5da188f5 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -97,58 +97,8 @@ pub(crate) fn select_account( Ok(AccountInfo { summary, details }) } -/// Select account info at a specific block number from the DB using the given -/// [`SqliteConnection`]. -/// -/// # Returns -/// -/// The account info at the specified block, or an error. -/// -/// # Note -/// -/// This function returns only the account summary (id, commitment, `block_num`). -/// Full account details are no longer reconstructed here - use separate query functions -/// to fetch specific account components as needed. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// accounts.account_id, -/// accounts.account_commitment, -/// accounts.block_num -/// FROM -/// accounts -/// WHERE -/// account_id = ?1 -/// AND block_num = ?2 -/// ``` -pub(crate) fn select_historical_account_at( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result { - let raw = SelectDsl::select(schema::accounts::table, AccountSummaryRaw::as_select()) - .filter( - schema::accounts::account_id - .eq(account_id.to_bytes()) - .and(schema::accounts::block_num.eq(block_num.to_raw_sql())), - ) - .get_result::(conn) - .optional()? - .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; - - let summary: AccountSummary = raw.try_into()?; - - // Backfill account details from database (at the specific historical block) - // Note: We use `ok()` to convert errors to None, as historical data might not have full details - let details = reconstruct_full_account_from_db(conn, account_id).ok(); - - Ok(AccountInfo { summary, details }) -} - /// Select the latest account info by account ID prefix from the DB using the given -/// [`SqliteConnection`]. This method is meant to be used by the network transaction builder. +/// [`SqliteConnection`]. Meant to be used by the network transaction builder. /// Because network notes get matched through accounts through the account's 30-bit prefix, it is /// possible that multiple accounts match against a single prefix. In this scenario, the first /// account is returned. @@ -534,88 +484,6 @@ pub(crate) fn select_account_storage_map_values( Ok(StorageMapValuesPage { last_block_included, values }) } -/// Select specific storage map keys at a specific block from the DB using the given -/// [`SqliteConnection`]. -/// -/// This function queries the `account_storage_map_values` table for specific keys at or before -/// the given block number, avoiding the need to deserialize the entire account. -/// -/// # Arguments -/// -/// * `conn` - Database connection -/// * `account_id` - The account ID to query -/// * `block_num` - The block number to query at -/// * `slot_index` - The storage slot index -/// * `keys` - The specific keys to retrieve -/// -/// # Returns -/// -/// A vector of (key, value) tuples for the requested keys that exist in the storage map. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT DISTINCT -/// first_value(key) OVER w as key, -/// first_value(value) OVER w as value -/// FROM -/// account_storage_map_values -/// WHERE -/// account_id = ?1 -/// AND slot = ?2 -/// AND block_num <= ?3 -/// AND key IN (?4, ?5, ...) -/// WINDOW w AS ( -/// PARTITION BY key -/// ORDER BY block_num DESC -/// ) -/// ``` -pub(crate) fn select_storage_map_keys_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, - slot_index: u8, - keys: &[Word], -) -> Result, DatabaseError> { - use schema::account_storage_map_values as t; - - if keys.is_empty() { - return Ok(Vec::new()); - } - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - let slot_sql = slot_to_raw_sql(slot_index); - - // Convert keys to bytes for query - let keys_bytes: Vec> = - keys.iter().map(miden_objects::utils::Serializable::to_bytes).collect(); - - // Query for the requested keys at or before the specified block - let raw: Vec<(Vec, Vec)> = SelectDsl::select(t::table, (t::key, t::value)) - .filter( - t::account_id - .eq(&account_id_bytes) - .and(t::slot.eq(slot_sql)) - .and(t::block_num.le(block_num_sql)) - .and(t::key.eq_any(&keys_bytes)), - ) - .distinct() - .load(conn)?; - - // Parse results - let results: Vec<(Word, Word)> = raw - .into_iter() - .map(|(key_bytes, value_bytes)| { - let key = Word::read_from_bytes(&key_bytes)?; - let value = Word::read_from_bytes(&value_bytes)?; - Ok((key, value)) - }) - .collect::, DatabaseError>>()?; - - Ok(results) -} - /// Reconstruct a `StorageMap` from database entries using `SmtForest` /// /// This function builds an `SmtForest` from all key-value pairs at the specified block, @@ -938,13 +806,13 @@ pub(crate) fn insert_account_vault_asset( /// Insert an account storage header into the DB using the given [`SqliteConnection`]. /// -/// This function will set `is_latest=true` for the new row and update any existing +/// Sets `is_latest=true` for the new row and updates any existing /// row with the same `(account_id, slot_index)` tuple to `is_latest=false`. /// /// # Returns /// /// The number of affected rows. -#[allow(dead_code)] // Used in tests +#[cfg(test)] pub(crate) fn insert_account_storage_header( conn: &mut SqliteConnection, account_id: AccountId, diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 5fa760bae7..b424fde99a 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1656,59 +1656,7 @@ fn test_storage_reconstruction_historical_state() { } } -#[test] -#[miden_node_test_macro::enable_logging] -fn test_storage_map_specific_keys_query() { - let mut conn = create_db(); - - let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - let block_num = BlockNumber::from(1); - let slot_index = 0u8; - - // Insert storage map header - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num, - slot_index, - StorageSlotType::Map, - EMPTY_WORD, // placeholder commitment - ) - .unwrap(); - // Insert several map entries - for i in 1..=10 { - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block_num, - slot_index, - num_to_word(i), - num_to_word(i * 100), - ) - .unwrap(); - } - - // Query specific keys - let requested_keys = vec![num_to_word(2), num_to_word(5), num_to_word(8)]; - let results = queries::select_storage_map_keys_at_block( - &mut conn, - account_id, - block_num, - slot_index, - &requested_keys, - ) - .unwrap(); - - // Should return exactly 3 entries - assert_eq!(results.len(), 3); - - // Verify the values - let result_map: std::collections::HashMap<_, _> = results.into_iter().collect(); - assert_eq!(result_map.get(&num_to_word(2)), Some(&num_to_word(200))); - assert_eq!(result_map.get(&num_to_word(5)), Some(&num_to_word(500))); - assert_eq!(result_map.get(&num_to_word(8)), Some(&num_to_word(800))); -} #[test] #[miden_node_test_macro::enable_logging] From ccc2d639d816d43555930e52cecfe7339cc07517 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 2 Dec 2025 19:43:26 +0100 Subject: [PATCH 15/95] sanitize comments --- crates/store/src/db/models/queries/accounts.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 3c5da188f5..de30e67567 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -607,7 +607,7 @@ pub(crate) fn select_account_storage_at_block( /// Select account storage headers at a specific block (lightweight query). /// -/// Returns tuples of (`slot_index`, `slot_type`, `commitment`) without reconstructing full slots. +/// Returns tuples of `(slot_index, slot_type, commitment)` without reconstructing full slots. #[allow(dead_code)] // Helper for future SmtForest integration pub(crate) fn select_account_storage_headers_at_block( conn: &mut SqliteConnection, @@ -766,8 +766,8 @@ impl TryInto for AccountSummaryRaw { /// Insert an account vault asset row into the DB using the given [`SqliteConnection`]. /// -/// This function will set `is_latest=true` for the new row and update any existing -/// row with the same `(account_id, vault_key)` tuple to `is_latest=false`. +/// Sets `is_latest=true` for the new row and updates any existing +/// row with the same (account_id, vault_key) tuple to `is_latest=false`. /// /// # Returns /// @@ -807,7 +807,7 @@ pub(crate) fn insert_account_vault_asset( /// Insert an account storage header into the DB using the given [`SqliteConnection`]. /// /// Sets `is_latest=true` for the new row and updates any existing -/// row with the same `(account_id, slot_index)` tuple to `is_latest=false`. +/// row with the same (account_id, slot_index) tuple to `is_latest=false`. /// /// # Returns /// @@ -857,10 +857,10 @@ pub(crate) fn insert_account_storage_header( }) } -/// Insert an account storage header into the DB using the given [`SqliteConnection`]. +/// Insert an account storage map value into the DB using the given [`SqliteConnection`]. /// -/// This function will set `is_latest=true` for the new row and update any existing -/// row with the same `(account_id, slot_index)` tuple to `is_latest=false`. +/// Sets `is_latest=true` for the new row and updates any existing +/// row with the same (account_id, slot_index, key) tuple to `is_latest=false`. /// /// # Returns /// From 66ea8311264d9d75148940f0aeadf82cc15134b1 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 4 Dec 2025 23:38:31 +0100 Subject: [PATCH 16/95] remove --- .../src/db/migrations/2025062000000_setup/down.sql | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/crates/store/src/db/migrations/2025062000000_setup/down.sql b/crates/store/src/db/migrations/2025062000000_setup/down.sql index da665f5668..e69de29bb2 100644 --- a/crates/store/src/db/migrations/2025062000000_setup/down.sql +++ b/crates/store/src/db/migrations/2025062000000_setup/down.sql @@ -1,12 +0,0 @@ --- Drop all tables in reverse order of creation (respecting foreign key dependencies) -DROP TABLE IF EXISTS transactions; -DROP TABLE IF EXISTS nullifiers; -DROP TABLE IF EXISTS account_vault_headers; -DROP TABLE IF EXISTS account_vault_assets; -DROP TABLE IF EXISTS account_storage_map_values; -DROP TABLE IF EXISTS note_scripts; -DROP TABLE IF EXISTS notes; -DROP TABLE IF EXISTS account_storage_headers; -DROP TABLE IF EXISTS accounts; -DROP TABLE IF EXISTS account_codes; -DROP TABLE IF EXISTS block_headers; From 1c4f8b18ad0fd1bc47f8a37ccd8b25a2c1de4ed2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 5 Dec 2025 00:29:20 +0100 Subject: [PATCH 17/95] cleanup --- crates/store/src/db/mod.rs | 2 -- .../store/src/db/models/queries/accounts.rs | 20 +++++-------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 653505ebec..3eeca2d5f9 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -409,8 +409,6 @@ impl Db { .await } - - /// Loads public account details from the DB based on the account ID's prefix. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_network_account_by_prefix( diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index de30e67567..7a0c8268cc 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -730,16 +730,6 @@ pub struct AccountStorageHeaderRaw { pub is_latest: bool, } -#[derive(Debug, Clone, Queryable, QueryableByName, Selectable)] -#[diesel(table_name = schema::accounts)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -pub struct AccountRaw { - pub account_id: Vec, - pub account_commitment: Vec, - pub block_num: i64, - pub nonce: Option, -} - #[derive(Debug, Clone, PartialEq, Eq, Selectable, Queryable, QueryableByName)] #[diesel(table_name = schema::accounts)] #[diesel(check_for_backend(Sqlite))] @@ -767,7 +757,7 @@ impl TryInto for AccountSummaryRaw { /// Insert an account vault asset row into the DB using the given [`SqliteConnection`]. /// /// Sets `is_latest=true` for the new row and updates any existing -/// row with the same (account_id, vault_key) tuple to `is_latest=false`. +/// row with the same `(account_id, vault_key)` tuple to `is_latest=false`. /// /// # Returns /// @@ -860,7 +850,7 @@ pub(crate) fn insert_account_storage_header( /// Insert an account storage map value into the DB using the given [`SqliteConnection`]. /// /// Sets `is_latest=true` for the new row and updates any existing -/// row with the same (account_id, slot_index, key) tuple to `is_latest=false`. +/// row with the same `(account_id, slot_index, key)` tuple to `is_latest=false`. /// /// # Returns /// @@ -925,9 +915,9 @@ fn reconstruct_full_account_from_db( account_id: AccountId, ) -> Result { // Get account metadata (nonce, code_commitment) and code in a single join query - let (account_raw, code_bytes): (AccountRaw, Vec) = SelectDsl::select( + let (nonce, code_bytes): (Option, Vec) = SelectDsl::select( schema::accounts::table.inner_join(schema::account_codes::table), - (AccountRaw::as_select(), schema::account_codes::code), + (schema::accounts::nonce, schema::account_codes::code), ) .filter(schema::accounts::account_id.eq(account_id.to_bytes())) .filter(schema::accounts::is_latest.eq(true)) @@ -935,7 +925,7 @@ fn reconstruct_full_account_from_db( .optional()? .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; - let nonce = raw_sql_to_nonce(account_raw.nonce.ok_or_else(|| { + let nonce = raw_sql_to_nonce(nonce.ok_or_else(|| { DatabaseError::DataCorrupted(format!("No nonce found for account {account_id}")) })?); From 80e0393cdc4f6ae5957f5d707334b0d64f9cc4b4 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 5 Dec 2025 00:30:13 +0100 Subject: [PATCH 18/95] fix queries with _at suffix --- .../store/src/db/models/queries/accounts.rs | 31 +++++++++++-------- crates/store/src/db/tests.rs | 2 -- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 7a0c8268cc..953afa8d8c 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -1219,11 +1219,12 @@ pub(crate) fn select_account_vault_at_block( let account_id_bytes = account_id.to_bytes(); let block_num_sql = i64::from(block_num.as_u32()); - let raw: Vec<(Vec, Option>)> = SelectDsl::select( t::table .filter(t::account_id.eq(&account_id_bytes)) - .filter(t::block_num.eq(block_num_sql)), + .filter(t::block_num.le(block_num_sql)) + .order(t::block_num.desc()) + .limit(1), (t::vault_key, t::asset), ) .load(conn)?; @@ -1287,14 +1288,15 @@ pub(crate) fn select_account_code_at_block( let account_id_bytes = account_id.to_bytes(); let block_num_sql = i64::from(block_num.as_u32()); - - // Query the accounts table to get the code_commitment at the specified block + // Query the accounts table to get the code_commitment at the specified block or earlier // Then join with account_codes to get the actual code let result: Option> = SelectDsl::select( accounts::table .inner_join(account_codes::table) .filter(accounts::account_id.eq(&account_id_bytes)) - .filter(accounts::block_num.eq(block_num_sql)), + .filter(accounts::block_num.le(block_num_sql)) + .order(accounts::block_num.desc()) + .limit(1), account_codes::code, ) .first(conn) @@ -1305,8 +1307,8 @@ pub(crate) fn select_account_code_at_block( /// Queries the account header for a specific account at a specific block number. /// -/// This reconstructs the AccountHeader by joining multiple tables: -/// - `accounts` table for account_id, nonce, `code_commitment` +/// This reconstructs the `AccountHeader` by joining multiple tables: +/// - `accounts` table for `account_id`, `nonce`, `code_commitment` /// - `account_vault_headers` table for `vault_root` /// - `account_storage_headers` table for storage slot commitments (to compute `storage_commitment`) /// @@ -1332,11 +1334,12 @@ pub(crate) fn select_account_header_at_block( let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); - let account_data: Option<(Option>, Option)> = SelectDsl::select( accounts::table .filter(accounts::account_id.eq(&account_id_bytes)) - .filter(accounts::block_num.eq(block_num_sql)), + .filter(accounts::block_num.le(block_num_sql)) + .order(accounts::block_num.desc()) + .limit(1), (accounts::code_commitment, accounts::nonce), ) .first(conn) @@ -1345,11 +1348,12 @@ pub(crate) fn select_account_header_at_block( let Some((code_commitment_bytes, nonce_raw)) = account_data else { return Ok(None); }; - let vault_root_bytes: Option> = SelectDsl::select( account_vault_headers::table .filter(account_vault_headers::account_id.eq(&account_id_bytes)) - .filter(account_vault_headers::block_num.eq(block_num_sql)), + .filter(account_vault_headers::block_num.le(block_num_sql)) + .order(account_vault_headers::block_num.desc()) + .limit(1), account_vault_headers::vault_root, ) .first(conn) @@ -1358,8 +1362,9 @@ pub(crate) fn select_account_header_at_block( let storage_slots: Vec<(i32, i32, Vec)> = SelectDsl::select( account_storage_headers::table .filter(account_storage_headers::account_id.eq(&account_id_bytes)) - .filter(account_storage_headers::block_num.eq(block_num_sql)) - .order(account_storage_headers::slot_index.asc()), + .filter(account_storage_headers::block_num.le(block_num_sql)) + .order(account_storage_headers::block_num.desc()) + .limit(1), ( account_storage_headers::slot_index, account_storage_headers::slot_type, diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index b424fde99a..499fc2a93c 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1656,8 +1656,6 @@ fn test_storage_reconstruction_historical_state() { } } - - #[test] #[miden_node_test_macro::enable_logging] fn test_storage_reconstruction_latest() { From e7bf1aa3e3dacb84deeda9bde8e1a6eaecc713d8 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 5 Dec 2025 00:30:28 +0100 Subject: [PATCH 19/95] cleanup --- crates/store/src/db/models/queries/accounts.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 953afa8d8c..a8f86059bf 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -1259,7 +1259,7 @@ fn compute_storage_commitment(slot_commitments: &[Word]) -> Word { let elements: Vec = slot_commitments.iter().flat_map(|w| w.iter()).copied().collect(); - Rpo256::hash_elements(&elements).into() + Rpo256::hash_elements(&elements) } /// Queries the account code for a specific account at a specific block number. From dad90e7ae97d12ce1756e7e567b7a4f5cb7357a8 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 5 Dec 2025 00:31:47 +0100 Subject: [PATCH 20/95] simplify --- crates/store/src/state.rs | 188 ++++++++++++++------------------------ 1 file changed, 69 insertions(+), 119 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index fdcef630aa..db3f8d99c5 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -51,7 +51,7 @@ use miden_objects::crypto::merkle::{ use miden_objects::note::{NoteDetails, NoteId, NoteScript, Nullifier}; use miden_objects::transaction::{OutputNote, PartialBlockchain}; use miden_objects::utils::Serializable; -use miden_objects::{AccountError, Word}; +use miden_objects::{AccountError, EMPTY_WORD, Word}; use tokio::sync::{Mutex, RwLock, oneshot}; use tracing::{info, info_span, instrument}; @@ -91,6 +91,31 @@ pub struct TransactionInputs { pub new_account_id_prefix_is_unique: Option, } +/// Container for forest-related state that needs to be updated atomically. +struct InnerForest { + /// `SmtForest` for efficient account storage reconstruction. + /// Populated during block import with storage and vault SMTs. + storage_forest: SmtForest, + + /// Maps (`account_id`, `slot_index`, `block_num`) to SMT root. + /// Populated during block import for all storage map slots. + storage_roots: BTreeMap<(AccountId, u8, BlockNumber), Word>, + + /// Maps (`account_id`, `block_num`) to vault SMT root. + /// Tracks asset vault versions across all blocks with structural sharing. + vault_roots: BTreeMap<(AccountId, BlockNumber), Word>, +} + +impl InnerForest { + fn new() -> Self { + Self { + storage_forest: SmtForest::new(), + storage_roots: BTreeMap::new(), + vault_roots: BTreeMap::new(), + } + } +} + /// Container for state that needs to be updated atomically. struct InnerState where @@ -127,21 +152,12 @@ pub struct State { /// The lock is writer-preferring, meaning the writer won't be starved. inner: RwLock, + /// Forest-related state `(SmtForest, storage_roots, vault_roots)` with its own lock. + forest: RwLock, + /// To allow readers to access the tree data while an update in being performed, and prevent /// TOCTOU issues, there must be no concurrent writers. This locks to serialize the writers. writer: Mutex<()>, - - /// `SmtForest` for efficient account storage reconstruction. - /// Populated during block import with storage and vault SMTs. - storage_forest: RwLock, - - /// Maps (`account_id`, `slot_index`, `block_num`) to SMT root. - /// Populated during block import for all storage map slots. - storage_roots: RwLock>, - - /// Maps (`account_id`, `block_num`) to vault SMT root. - /// Tracks asset vault versions across all blocks with structural sharing. - vault_roots: RwLock>, } impl State { @@ -177,22 +193,11 @@ impl State { account_tree, }); + let forest = RwLock::new(InnerForest::new()); let writer = Mutex::new(()); let db = Arc::new(db); - let storage_forest = RwLock::new(SmtForest::new()); - let storage_roots = RwLock::new(BTreeMap::new()); - let vault_roots = RwLock::new(BTreeMap::new()); - - let me = Self { - db, - block_store, - inner, - writer, - storage_forest, - storage_roots, - vault_roots, - }; + let me = Self { db, block_store, inner, forest, writer }; // load all accounts from the table // TODO: make `select_all_account_at(block_num)` to be precise; if ACID is upheld, it's not @@ -559,9 +564,9 @@ impl State { // Iterate through each slot in the account storage for (slot_idx, slot) in storage.slots().iter().enumerate() { // Only process Map-type slots - if let miden_objects::account::StorageSlot::Map(storage_map) = slot { + if let StorageSlot::Map(storage_map) = slot { // Extract all (key, value) entries from this StorageMap - let entries: Vec<_> = storage_map.entries().collect(); + let entries = Vec::from_iter(storage_map.entries()); tracing::debug!( target: COMPONENT, @@ -583,31 +588,25 @@ impl State { "Successfully extracted Map slots and entries (Step 2 complete)" ); - // STEP 3: Get previous roots from storage_roots or use empty root - let storage_roots = self.storage_roots.read().await; - let prev_block_num = if block_num.as_u32() > 0 { - BlockNumber::from(block_num.as_u32() - 1) - } else { - // Genesis block - no previous block - block_num - }; + // Acquire a single write lock on the forest for the entire update operation. + // Since apply_block() is already serialized by the `writer` Mutex, holding this lock + // for the entire duration is acceptable and simplifies the code by avoiding multiple + // lock acquisitions. + let mut forest_guard = self.forest.write().await; - // For each map slot, get the previous root or use empty root - let mut slots_with_prev_roots = Vec::new(); + let prev_block_num = block_num.parent().unwrap_or_default(); + // STEP 3 & 4 & 5: Process each map slot: get previous root, build new SMT, track new root for (account_id, slot_idx, entries) in map_slots_to_populate { // Look up previous root for this (account_id, slot_idx, prev_block) let prev_root = if block_num.as_u32() > 0 { - storage_roots + forest_guard + .storage_roots .get(&(account_id, slot_idx, prev_block_num)) .copied() - .unwrap_or_else(|| { - // No previous root found, use empty SMT root - *EmptySubtreeRoots::entry(SMT_DEPTH, 0) - }) + .unwrap_or(EMPTY_WORD) } else { - // Genesis block - use empty root - *EmptySubtreeRoots::entry(SMT_DEPTH, 0) + EMPTY_WORD }; tracing::debug!( @@ -617,28 +616,11 @@ impl State { "Retrieved previous root for slot" ); - slots_with_prev_roots.push((account_id, slot_idx, prev_root, entries)); - } - - drop(storage_roots); // Release read lock before write operations - - tracing::info!( - target: COMPONENT, - %block_num, - num_slots = slots_with_prev_roots.len(), - "Successfully retrieved previous roots (Step 3 complete)" - ); - - // STEP 4: Use forest.insert(prev_root, key, value) to build new SMTs - let mut forest = self.storage_forest.write().await; - let mut new_roots = Vec::new(); - - for (account_id, slot_idx, prev_root, entries) in slots_with_prev_roots { - let updated_root = forest + // Use forest.batch_insert to build new SMT + let updated_root = forest_guard + .storage_forest .batch_insert(prev_root, entries.into_iter().map(|(k, v)| (*k, *v))) .expect("Insertion into Forest always works"); - // Store the final root after all insertions - new_roots.push((account_id, slot_idx, updated_root)); tracing::debug!( target: COMPONENT, @@ -646,23 +628,11 @@ impl State { slot_index = slot_idx, "Built new SMT in forest" ); - } - - drop(forest); // Release write lock before next write - - tracing::info!( - target: COMPONENT, - %block_num, - num_new_roots = new_roots.len(), - "Successfully built new SMTs in forest (Step 4 complete)" - ); - - // STEP 5: Track new roots in storage_roots map - let mut storage_roots = self.storage_roots.write().await; - for (account_id, slot_idx, new_root) in new_roots { - // Insert the new root for this (account_id, slot_idx, block_num) triple - storage_roots.insert((account_id, slot_idx, block_num), new_root); + // Track the new root for this (account_id, slot_idx, block_num) triple + forest_guard + .storage_roots + .insert((account_id, slot_idx, block_num), updated_root); tracing::debug!( target: COMPONENT, @@ -676,8 +646,8 @@ impl State { tracing::info!( target: COMPONENT, %block_num, - total_tracked_roots = storage_roots.len(), - "Successfully tracked new roots (Step 5 complete)" + total_tracked_roots = forest_guard.storage_roots.len(), + "Successfully completed storage map SMT updates" ); Ok(()) @@ -701,7 +671,7 @@ impl State { let mut vault_entries_to_populate = Vec::new(); for &account_id in changed_account_ids { - match self.db.select_account_vault_at_block(account_id, block_num).await { + match self.db.select_account_vault_at_block(account_id, block_num).await? { Ok(entries) if !entries.is_empty() => { vault_entries_to_populate.push((account_id, entries)); }, @@ -725,18 +695,18 @@ impl State { "Queried vault assets" ); - // Get previous vault roots - let vault_roots_read = self.vault_roots.read().await; - let prev_block_num = if block_num.as_u32() > 0 { - BlockNumber::from(block_num.as_u32() - 1) - } else { - block_num - }; + // Acquire a single write lock on the forest for the entire update operation. + // Since apply_block() is already serialized by the `writer` Mutex, holding this lock + // for the entire duration is acceptable and simplifies the code. + let mut forest_guard = self.forest.write().await; - let mut vaults_with_prev_roots = Vec::new(); + let prev_block_num = block_num.parent().unwrap_or_default(); + + // Process each vault: get previous root, build new SMT, track new root for (account_id, entries) in vault_entries_to_populate { let prev_root = if block_num.as_u32() > 0 { - vault_roots_read + forest_guard + .vault_roots .get(&(account_id, prev_block_num)) .copied() .unwrap_or_else(|| *EmptySubtreeRoots::entry(SMT_DEPTH, 0)) @@ -744,40 +714,20 @@ impl State { *EmptySubtreeRoots::entry(SMT_DEPTH, 0) }; - vaults_with_prev_roots.push((account_id, prev_root, entries)); - } - drop(vault_roots_read); - - // Build vault SMTs in forest - let mut forest = self.storage_forest.write().await; - let mut vault_new_roots = Vec::new(); - - for (account_id, prev_root, entries) in vaults_with_prev_roots { - let updated_root = forest + let updated_root = forest_guard + .storage_forest .batch_insert(prev_root, entries) .expect("Database is consistent and always allows constructing a smt or forest"); - vault_new_roots.push((account_id, updated_root)); - } - drop(forest); - - tracing::info!( - target: COMPONENT, - %block_num, - num_vault_roots = vault_new_roots.len(), - "Built vault SMTs in forest" - ); - // Track vault roots - let mut vault_roots = self.vault_roots.write().await; - for (account_id, new_root) in vault_new_roots { - vault_roots.insert((account_id, block_num), new_root); + // Track the new vault root + forest_guard.vault_roots.insert((account_id, block_num), updated_root); } tracing::info!( target: COMPONENT, %block_num, - total_vault_roots = vault_roots.len(), - "Successfully tracked vault roots (Vault tracking complete)" + total_vault_roots = forest_guard.vault_roots.len(), + "Successfully completed vault SMT updates" ); Ok(()) From e441245e8f9cb82e110aff94836be67110737e63 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 5 Dec 2025 00:44:18 +0100 Subject: [PATCH 21/95] fix --- crates/store/src/state.rs | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index db3f8d99c5..d1ebb2cb4e 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -209,8 +209,7 @@ impl State { .await .map_err(|e| { StateInitializationError::DatabaseError(DatabaseError::InteractError(format!( - "Failed to update storage forest: {}", - e + "Failed to update storage forest: {e}" ))) })?; @@ -534,8 +533,6 @@ impl State { changed_account_ids: &[AccountId], block_num: BlockNumber, ) -> Result<(), ApplyBlockError> { - use miden_objects::crypto::merkle::{EmptySubtreeRoots, SMT_DEPTH}; - tracing::debug!( target: COMPONENT, %block_num, @@ -671,16 +668,11 @@ impl State { let mut vault_entries_to_populate = Vec::new(); for &account_id in changed_account_ids { - match self.db.select_account_vault_at_block(account_id, block_num).await? { - Ok(entries) if !entries.is_empty() => { - vault_entries_to_populate.push((account_id, entries)); - }, - Ok(_) => { - tracing::debug!(%account_id, "Account has empty vault"); - }, - Err(e) => { - tracing::warn!(%account_id, error = %e, "Failed to query vault assets"); - }, + let entries = self.db.select_account_vault_at_block(account_id, block_num).await?; + if entries.is_empty() { + tracing::debug!(%account_id, "Account has empty vault"); + } else { + vault_entries_to_populate.push((account_id, entries)); } } From 0a319d1e3520c3c1231afd0ffe74f2f05dad9e5b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 5 Dec 2025 00:46:49 +0100 Subject: [PATCH 22/95] cleanup --- crates/store/src/db/models/queries/accounts.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index a8f86059bf..6cdc0dee6c 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -28,6 +28,7 @@ use miden_objects::account::{ AccountId, AccountStorage, NonFungibleDeltaAction, + StorageMap, StorageSlot, StorageSlotType, }; @@ -574,26 +575,21 @@ pub(crate) fn select_account_storage_at_block( let mut slots = Vec::with_capacity(headers.len()); for header in headers { - let slot_type = match header.slot_type { - 0 => miden_objects::account::StorageSlotType::Map, - 1 => miden_objects::account::StorageSlotType::Value, - _ => return Err(DatabaseError::InvalidStorageSlotType(header.slot_type)), - }; + let slot_type = StorageSlotType::from_raw_sql(header.slot_type)?; let commitment = Word::read_from_bytes(&header.slot_commitment)?; let slot = match slot_type { - miden_objects::account::StorageSlotType::Map => { + StorageSlotType::Map => { // For Map slots, we create an empty map // The actual map data is queried separately when needed from // account_storage_map_values - use miden_objects::account::StorageMap; // Create an empty storage map let storage_map = StorageMap::new(); StorageSlot::Map(storage_map) }, - miden_objects::account::StorageSlotType::Value => { + StorageSlotType::Value => { // For Value slots, the commitment IS the value StorageSlot::Value(commitment) }, From 8897939a02dc0069e5d130017303835d3705343b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 5 Dec 2025 00:51:11 +0100 Subject: [PATCH 23/95] remove dead code --- crates/proto/src/domain/account.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index a70022ea14..a392aa8fc4 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -471,16 +471,6 @@ impl AccountStorageMapDetails { } } - // TODO this is - #[allow(dead_code)] - fn from_specific_keys(slot_index: u8, keys: &[Word], _storage_map: &StorageMap) -> Self { - if keys.len() > Self::MAX_RETURN_ENTRIES { - Self::too_many_entries(slot_index) - } else { - todo!("construct a partial SMT / set of key values") - } - } - /// Creates an `AccountStorageMapDetails` from already-queried entries (e.g., from database). /// This is useful when entries have been fetched directly rather than extracted from a /// `StorageMap`. From 7d7fefc635fbdb6d627104ff0b61a220b10cc894 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 5 Dec 2025 08:40:42 +0100 Subject: [PATCH 24/95] add test --- crates/store/src/db/tests.rs | 136 +++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 499fc2a93c..2998b2030d 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1851,6 +1851,9 @@ fn test_select_account_code_at_block() { let block_num_1 = BlockNumber::from(1); let block_num_2 = BlockNumber::from(2); + // Create block 1 + create_block(&mut conn, block_num_1); + // Create an account with code at block 1 using the existing mock function let account = mock_account_code_and_storage( AccountType::RegularAccountImmutableCode, @@ -1894,3 +1897,136 @@ fn test_select_account_code_at_block() { queries::select_account_code_at_block(&mut conn, other_account_id, block_num_1).unwrap(); assert!(code_other.is_none(), "Code should not exist for non-existent account"); } + +#[test] +fn test_select_account_code_at_block_with_updates() { + let mut conn = create_db(); + + let block_num_1 = BlockNumber::from(1); + let block_num_2 = BlockNumber::from(2); + let block_num_3 = BlockNumber::from(3); + + // Create all blocks + create_block(&mut conn, block_num_1); + create_block(&mut conn, block_num_2); + create_block(&mut conn, block_num_3); + + // Helper function to create account with specific code + fn create_account_with_code(code_str: &str, seed: [u8; 32]) -> Account { + let component_storage = vec![ + StorageSlot::Value(Word::empty()), + StorageSlot::Value(num_to_word(1)), + ]; + + let component = AccountComponent::compile( + code_str, + TransactionKernel::assembler(), + component_storage, + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountUpdatableCode); + + AccountBuilder::new(seed) + .account_type(AccountType::RegularAccountUpdatableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap() + } + + // Create initial account with code v1 at block 1 + let code_v1_str = "\ + export.account_procedure_1 + push.1.2 + add + end + "; + let account_v1 = create_account_with_code(code_v1_str, [1u8; 32]); + let account_id = account_v1.id(); + let code_v1 = account_v1.code().to_bytes(); + + // Insert the account at block 1 + queries::upsert_accounts( + &mut conn, + &[BlockAccountUpdate::new( + account_id, + account_v1.commitment(), + AccountUpdateDetails::Delta(AccountDelta::try_from(account_v1).unwrap()), + )], + block_num_1, + ) + .unwrap(); + + // Create account with different code v2 at block 2 + let code_v2_str = "\ + export.account_procedure_1 + push.3.4 + mul + end + "; + let account_v2 = create_account_with_code(code_v2_str, [1u8; 32]); // Same seed to keep same account_id + let code_v2 = account_v2.code().to_bytes(); + + // Verify that the codes are actually different + assert_ne!( + code_v1, code_v2, + "Test setup error: codes should be different for different code strings" + ); + + // Insert the updated account at block 2 + queries::upsert_accounts( + &mut conn, + &[BlockAccountUpdate::new( + account_id, + account_v2.commitment(), + AccountUpdateDetails::Delta(AccountDelta::try_from(account_v2).unwrap()), + )], + block_num_2, + ) + .unwrap(); + + // Create account with different code v3 at block 3 + let code_v3_str = "\ + export.account_procedure_1 + push.5.6 + sub + end + "; + let account_v3 = create_account_with_code(code_v3_str, [1u8; 32]); // Same seed to keep same account_id + let code_v3 = account_v3.code().to_bytes(); + + // Verify that v3 code is different from v2 and v1 + assert_ne!(code_v2, code_v3, "Test setup error: v3 code should differ from v2"); + assert_ne!(code_v1, code_v3, "Test setup error: v3 code should differ from v1"); + + // Insert the updated account at block 3 + queries::upsert_accounts( + &mut conn, + &[BlockAccountUpdate::new( + account_id, + account_v3.commitment(), + AccountUpdateDetails::Delta(AccountDelta::try_from(account_v3).unwrap()), + )], + block_num_3, + ) + .unwrap(); + + // Test: Query code at block 1 - should return v1 code + let code_at_1 = queries::select_account_code_at_block(&mut conn, account_id, block_num_1) + .unwrap() + .expect("Code should exist at block 1"); + assert_eq!(code_at_1, code_v1, "Block 1 should return v1 code"); + + // Test: Query code at block 2 - should return v2 code (even though we're at block 3) + let code_at_2 = queries::select_account_code_at_block(&mut conn, account_id, block_num_2) + .unwrap() + .expect("Code should exist at block 2"); + assert_eq!(code_at_2, code_v2, "Block 2 should return v2 code"); + + // Test: Query code at block 3 - should return v3 code + let code_at_3 = queries::select_account_code_at_block(&mut conn, account_id, block_num_3) + .unwrap() + .expect("Code should exist at block 3"); + assert_eq!(code_at_3, code_v3, "Block 3 should return v3 code"); +} From 0f53fa9a194adf6236f756e26c4a1d3b81d5be4b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 5 Dec 2025 09:56:54 +0100 Subject: [PATCH 25/95] add block exists helper --- .../store/src/db/models/queries/accounts.rs | 46 +++++++++++++++++++ crates/store/src/db/tests.rs | 43 +++++++++-------- crates/store/src/errors.rs | 2 + 3 files changed, 73 insertions(+), 18 deletions(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 6cdc0dee6c..76d8ca579e 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -509,6 +509,9 @@ pub(crate) fn reconstruct_storage_map_at_block( ) -> Result { use schema::account_storage_map_values as t; + // Check if the requested block exists (returns error if not) + block_exists(conn, block_num)?; + let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); let slot_sql = slot_to_raw_sql(slot_index); @@ -556,6 +559,9 @@ pub(crate) fn select_account_storage_at_block( ) -> Result { use schema::account_storage_headers as t; + // Check if the requested block exists (returns error if not) + block_exists(conn, block_num)?; + let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); @@ -1213,6 +1219,9 @@ pub(crate) fn select_account_vault_at_block( ) -> Result, DatabaseError> { use schema::account_vault_assets as t; + // Check if the requested block exists (returns error if not) + block_exists(conn, block_num)?; + let account_id_bytes = account_id.to_bytes(); let block_num_sql = i64::from(block_num.as_u32()); let raw: Vec<(Vec, Option>)> = SelectDsl::select( @@ -1258,6 +1267,37 @@ fn compute_storage_commitment(slot_commitments: &[Word]) -> Word { Rpo256::hash_elements(&elements) } +/// Helper function to check if a block exists in the block_headers table. +/// +/// This should be called by all `_at_block` query functions to ensure that +/// queries are only performed against blocks that have been produced. +/// +/// # Arguments +/// +/// * `conn` - Database connection +/// * `block_num` - The block number to check +/// +/// # Returns +/// +/// * `Ok(())` - If the block exists +/// * `Err(DatabaseError::BlockNotFound)` - If the block doesn't exist +/// * `Err(DatabaseError)` - If there's a database error +fn block_exists(conn: &mut SqliteConnection, block_num: BlockNumber) -> Result<(), DatabaseError> { + use schema::block_headers; + + let count: i64 = SelectDsl::select( + block_headers::table.filter(block_headers::block_num.eq(block_num.to_raw_sql())), + diesel::dsl::count(block_headers::block_num), + ) + .first(conn)?; + + if count > 0 { + Ok(()) + } else { + Err(DatabaseError::BlockNotFound(block_num)) + } +} + /// Queries the account code for a specific account at a specific block number. /// /// Returns `None` if: @@ -1282,6 +1322,9 @@ pub(crate) fn select_account_code_at_block( ) -> Result>, DatabaseError> { use schema::{account_codes, accounts}; + // Check if the requested block exists (returns error if not) + block_exists(conn, block_num)?; + let account_id_bytes = account_id.to_bytes(); let block_num_sql = i64::from(block_num.as_u32()); // Query the accounts table to get the code_commitment at the specified block or earlier @@ -1328,6 +1371,9 @@ pub(crate) fn select_account_header_at_block( ) -> Result, DatabaseError> { use schema::{account_storage_headers, account_vault_headers, accounts}; + // Check if the requested block exists (returns error if not) + block_exists(conn, block_num)?; + let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); let account_data: Option<(Option>, Option)> = SelectDsl::select( diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 2998b2030d..d912e317ee 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1527,6 +1527,9 @@ fn test_storage_reconstruction_latest_state() { let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); let block_num = BlockNumber::from(1); + // Create the block before inserting storage data + create_block(&mut conn, block_num); + // Create test storage with Value and Map slots let value_slot = StorageSlot::Value(num_to_word(42)); let mut storage_map = StorageMap::new(); @@ -1610,6 +1613,8 @@ fn test_storage_reconstruction_historical_state() { // Block 1: Initial storage let block_num_1 = BlockNumber::from(1); + create_block(&mut conn, block_num_1); + queries::insert_account_storage_header( &mut conn, account_id, @@ -1622,6 +1627,8 @@ fn test_storage_reconstruction_historical_state() { // Block 2: Updated storage let block_num_2 = BlockNumber::from(2); + create_block(&mut conn, block_num_2); + queries::insert_account_storage_header( &mut conn, account_id, @@ -1664,6 +1671,9 @@ fn test_storage_reconstruction_latest() { let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); let block_num = BlockNumber::from(1); + // Create the block + create_block(&mut conn, block_num); + // Insert storage headers: 2 Map slots and 1 Value slot let map_commitment_1 = [Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]; let map_commitment_2 = [Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]; @@ -1725,6 +1735,8 @@ fn test_storage_reconstruction_historical() { // Block 1: Initial state with one value slot let block_1 = BlockNumber::from(1); + create_block(&mut conn, block_1); + let value_1 = [Felt::new(10), Felt::new(20), Felt::new(30), Felt::new(40)]; queries::insert_account_storage_header( &mut conn, @@ -1738,6 +1750,7 @@ fn test_storage_reconstruction_historical() { // Block 2: Update the value slot let block_2 = BlockNumber::from(2); + create_block(&mut conn, block_2); let value_2 = [Felt::new(50), Felt::new(60), Felt::new(70), Felt::new(80)]; queries::insert_account_storage_header( &mut conn, @@ -1791,6 +1804,11 @@ fn test_storage_header_is_latest_flag() { let value_2 = [Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]; let value_3 = [Felt::new(9), Felt::new(10), Felt::new(11), Felt::new(12)]; + // Create the blocks + create_block(&mut conn, BlockNumber::from(1)); + create_block(&mut conn, BlockNumber::from(2)); + create_block(&mut conn, BlockNumber::from(3)); + // Insert at block 1 queries::insert_account_storage_header( &mut conn, @@ -1849,7 +1867,6 @@ fn test_select_account_code_at_block() { let mut conn = create_db(); let block_num_1 = BlockNumber::from(1); - let block_num_2 = BlockNumber::from(2); // Create block 1 create_block(&mut conn, block_num_1); @@ -1886,11 +1903,6 @@ fn test_select_account_code_at_block() { .expect("Code should exist at block 1"); assert_eq!(code_at_1, expected_code); - // Query code at non-existent block - should return None - let code_at_2 = - queries::select_account_code_at_block(&mut conn, account_id, block_num_2).unwrap(); - assert!(code_at_2.is_none(), "Code should not exist at block 2"); - // Query code for non-existent account - should return None let other_account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); let code_other = @@ -1913,18 +1925,13 @@ fn test_select_account_code_at_block_with_updates() { // Helper function to create account with specific code fn create_account_with_code(code_str: &str, seed: [u8; 32]) -> Account { - let component_storage = vec![ - StorageSlot::Value(Word::empty()), - StorageSlot::Value(num_to_word(1)), - ]; - - let component = AccountComponent::compile( - code_str, - TransactionKernel::assembler(), - component_storage, - ) - .unwrap() - .with_supported_type(AccountType::RegularAccountUpdatableCode); + let component_storage = + vec![StorageSlot::Value(Word::empty()), StorageSlot::Value(num_to_word(1))]; + + let component = + AccountComponent::compile(code_str, TransactionKernel::assembler(), component_storage) + .unwrap() + .with_supported_type(AccountType::RegularAccountUpdatableCode); AccountBuilder::new(seed) .account_type(AccountType::RegularAccountUpdatableCode) diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 7ae319a36d..8e37a42a7f 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -98,6 +98,8 @@ pub enum DatabaseError { AccountNotFoundInDb(AccountId), #[error("account {0} state at block height {1} not found")] AccountAtBlockHeightNotFoundInDb(AccountId, BlockNumber), + #[error("block {0} not found in database")] + BlockNotFound(BlockNumber), #[error("historical block {block_num} not available: {reason}")] HistoricalBlockNotAvailable { block_num: BlockNumber, reason: String }, #[error("accounts {0:?} not found")] From 0e2d871ec7c97d6a4f28f47d2066ed600c0532a0 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 9 Dec 2025 17:37:24 +0100 Subject: [PATCH 26/95] simplify --- crates/store/src/state.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 58386480ad..1944388067 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -656,8 +656,6 @@ impl State { changed_account_ids: &[AccountId], block_num: BlockNumber, ) -> Result<(), ApplyBlockError> { - use miden_objects::crypto::merkle::{EmptySubtreeRoots, SMT_DEPTH}; - tracing::debug!( target: COMPONENT, %block_num, @@ -696,15 +694,11 @@ impl State { // Process each vault: get previous root, build new SMT, track new root for (account_id, entries) in vault_entries_to_populate { - let prev_root = if block_num.as_u32() > 0 { - forest_guard - .vault_roots - .get(&(account_id, prev_block_num)) - .copied() - .unwrap_or_else(|| *EmptySubtreeRoots::entry(SMT_DEPTH, 0)) - } else { - *EmptySubtreeRoots::entry(SMT_DEPTH, 0) - }; + let prev_root = forest_guard + .vault_roots + .get(&(account_id, prev_block_num)) + .copied() + .unwrap_or(EMPTY_WORD); let updated_root = forest_guard .storage_forest From f78103e2d0537822d81539c197b1db82979d09c1 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 9 Dec 2025 18:06:50 +0100 Subject: [PATCH 27/95] better docs --- crates/proto/src/domain/account.rs | 57 +++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 8097589e36..5215d53669 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -347,12 +347,30 @@ impl From for proto::account::AccountStorageHeader { } } +/// Account vault assets +/// +/// Represents a list of assets, if the number of assets is reasonably small, which +/// is currently set to 1000 for no particular reason. +/// +/// When an account contains a large number of assets, including all assets +/// in a single RPC response would create performance issues on client and server as +/// and consume quite a bit of bandwidth, besides requiring additional memory on +/// possibly low powered clients. +/// +/// Hence `too_many_assets` is returned, which is indicating to the client to use the dedicated `SyncAccountVault` RPC endpoint and do incremental retrieval #[derive(Debug, Clone, PartialEq, Eq)] pub struct AccountVaultDetails { + /// Flag indicating whether the vault has too many assets to return inline. + /// If `true`, clients must use `SyncAccountVault` endpoint instead. pub too_many_assets: bool, + + /// The assets in the vault. Empty if `too_many_assets` is `true`. pub assets: Vec, } + impl AccountVaultDetails { + /// Maximum number of vault entries that can be returned in a single response. + /// Accounts with more assets will have `too_many_assets = true` and empty `assets`. const MAX_RETURN_ENTRIES: usize = 1000; pub fn new(vault: &AssetVault) -> Self { @@ -433,18 +451,49 @@ impl From for proto::rpc::AccountVaultDetails { } } +/// Details about an account storage map slot, including overflow handling. +/// +/// ## Rationale for "Too Many Entries" Flag +/// +/// Similar to `AccountVaultDetails`, when a storage map contains many entries (> 1000), +/// returning all entries in a single RPC response creates performance issues: +/// - Large serialization/deserialization costs +/// - Network bandwidth saturation +/// - Client memory pressure +/// +/// When `too_many_entries` is `true`: +/// - The `map_entries` field is empty (no data included) +/// - Clients should use the dedicated `SyncStorageMaps` RPC endpoint +/// - That endpoint supports pagination and block range filtering +/// +/// ## Future Enhancement (TODO) +/// +/// Currently, when `too_many_entries = true`, we return an empty list. A future improvement +/// would return a **partial SMT** with: +/// - A subset of entries (e.g., most frequently accessed) +/// - Merkle proofs for those entries +/// - Inner node commitments +/// +/// This would allow clients to verify partial data cryptographically while still +/// signaling that more data exists. The reason this matters: if all leaf values are +/// included, one can reconstruct the entire SMT; if even one is missing, one cannot. +/// By providing proofs, we enable verification of partial data. #[derive(Debug, Clone, PartialEq, Eq)] pub struct AccountStorageMapDetails { pub slot_index: u8, + + /// Flag indicating whether the storage map has too many entries to return inline. + /// If `true`, clients must use `SyncStorageMaps` endpoint instead. pub too_many_entries: bool, - // TODO the following is only for the case when _all_ entries are included - // TODO for partials, we also need to provide merkle proofs / a partial SMT with inner nodes - // Reason: if all leaf values are included, one can reconstruct the entire SMT, if just one - // is missing one cannot + + /// The storage map entries (key-value pairs). Empty if `too_many_entries` is `true`. + /// TODO: For partial responses, also include Merkle proofs and inner SMT nodes. pub map_entries: Vec<(Word, Word)>, } impl AccountStorageMapDetails { + /// Maximum number of storage map entries that can be returned in a single response. + /// Maps with more entries will have `too_many_entries = true` and empty `map_entries`. pub const MAX_RETURN_ENTRIES: usize = 1000; pub fn new(slot_index: u8, slot_data: SlotData, storage_map: &StorageMap) -> Self { From ea05b0162131b82bdc61d034fe2a763243ba9188 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 9 Dec 2025 18:07:08 +0100 Subject: [PATCH 28/95] split long function in State --- crates/store/src/state.rs | 130 +++++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 58 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 1944388067..73d4e9735e 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -527,7 +527,6 @@ impl State { } /// Updates storage map SMTs in the forest for changed accounts - #[allow(clippy::too_many_lines)] async fn update_storage_maps_in_forest( &self, changed_account_ids: &[AccountId], @@ -537,32 +536,60 @@ impl State { target: COMPONENT, %block_num, num_accounts = changed_account_ids.len(), - "Querying account storage from DB to populate SmtForest" + "Updating storage maps in forest" ); - // Query full storage for each updated account at this block - let mut account_storages = Vec::new(); - for &account_id in changed_account_ids { + // Step 1: Query storage from database + let account_storages = + self.query_account_storages_from_db(changed_account_ids, block_num).await?; + + // Step 2: Extract map slots and their entries + let map_slots_to_populate = self.extract_map_slots_from_storage(&account_storages); + + // Step 3: Update the forest with new SMTs + self.populate_forest_with_storage_maps(map_slots_to_populate, block_num).await?; + + tracing::info!( + target: COMPONENT, + %block_num, + "Successfully completed storage map SMT updates" + ); + + Ok(()) + } + + /// Queries account storage data from the database for the given accounts at a specific block + async fn query_account_storages_from_db( + &self, + account_ids: &[AccountId], + block_num: BlockNumber, + ) -> Result, ApplyBlockError> { + let mut account_storages = Vec::with_capacity(account_ids.len()); + + for &account_id in account_ids { let storage = self.db.select_account_storage_at_block(account_id, block_num).await?; account_storages.push((account_id, storage)); } - tracing::info!( + tracing::debug!( target: COMPONENT, - %block_num, num_accounts = account_storages.len(), - "Successfully queried account storage from DB (Step 1 complete)" + "Queried account storage from database" ); - // STEP 2: Extract Map slots and their entries from account_storages - let mut map_slots_to_populate = Vec::new(); + Ok(account_storages) + } + + /// Extracts map-type storage slots and their entries from account storage data + fn extract_map_slots_from_storage( + &self, + account_storages: &[(AccountId, miden_objects::account::AccountStorage)], + ) -> Vec<(AccountId, u8, Vec<(&Word, &Word)>)> { + let mut map_slots = Vec::new(); - for (account_id, storage) in &account_storages { - // Iterate through each slot in the account storage + for (account_id, storage) in account_storages { for (slot_idx, slot) in storage.slots().iter().enumerate() { - // Only process Map-type slots if let StorageSlot::Map(storage_map) = slot { - // Extract all (key, value) entries from this StorageMap let entries = Vec::from_iter(storage_map.entries()); tracing::debug!( @@ -570,63 +597,52 @@ impl State { %account_id, slot_index = slot_idx, num_entries = entries.len(), - "Extracted Map slot entries" + "Extracted map slot entries" ); - map_slots_to_populate.push((*account_id, slot_idx as u8, entries)); + map_slots.push((*account_id, slot_idx as u8, entries)); } } } - tracing::info!( + tracing::debug!( target: COMPONENT, - %block_num, - num_map_slots = map_slots_to_populate.len(), - "Successfully extracted Map slots and entries (Step 2 complete)" + num_map_slots = map_slots.len(), + "Extracted all map slots from storage" ); - // Acquire a single write lock on the forest for the entire update operation. - // Since apply_block() is already serialized by the `writer` Mutex, holding this lock - // for the entire duration is acceptable and simplifies the code by avoiding multiple - // lock acquisitions. - let mut forest_guard = self.forest.write().await; + map_slots + } - let prev_block_num = block_num.parent().unwrap_or_default(); + /// Populates the forest with storage map SMTs for the given slots + async fn populate_forest_with_storage_maps( + &self, + map_slots: Vec<(AccountId, u8, Vec<(&Word, &Word)>)>, + block_num: BlockNumber, + ) -> Result<(), ApplyBlockError> { + if map_slots.is_empty() { + return Ok(()); + } - // STEP 3 & 4 & 5: Process each map slot: get previous root, build new SMT, track new root - for (account_id, slot_idx, entries) in map_slots_to_populate { - // Look up previous root for this (account_id, slot_idx, prev_block) - let prev_root = if block_num.as_u32() > 0 { - forest_guard - .storage_roots - .get(&(account_id, slot_idx, prev_block_num)) - .copied() - .unwrap_or(EMPTY_WORD) - } else { - EMPTY_WORD - }; + // Acquire write lock once for all updates + let mut forest_guard = self.forest.write().await; + let prev_block_num = block_num.parent().unwrap_or_default(); - tracing::debug!( - target: COMPONENT, - %account_id, - slot_index = slot_idx, - "Retrieved previous root for slot" - ); + for (account_id, slot_idx, entries) in map_slots { + // Get previous root for structural sharing + let prev_root = forest_guard + .storage_roots + .get(&(account_id, slot_idx, prev_block_num)) + .copied() + .unwrap_or(EMPTY_WORD); - // Use forest.batch_insert to build new SMT + // Build new SMT from entries let updated_root = forest_guard .storage_forest .batch_insert(prev_root, entries.into_iter().map(|(k, v)| (*k, *v))) - .expect("Insertion into Forest always works"); - - tracing::debug!( - target: COMPONENT, - %account_id, - slot_index = slot_idx, - "Built new SMT in forest" - ); + .expect("Forest insertion should always succeed with valid entries"); - // Track the new root for this (account_id, slot_idx, block_num) triple + // Track the new root forest_guard .storage_roots .insert((account_id, slot_idx, block_num), updated_root); @@ -635,16 +651,14 @@ impl State { target: COMPONENT, %account_id, slot_index = slot_idx, - %block_num, - "Tracked new root in storage_roots map" + "Updated storage map SMT in forest" ); } tracing::info!( target: COMPONENT, - %block_num, total_tracked_roots = forest_guard.storage_roots.len(), - "Successfully completed storage map SMT updates" + "Populated forest with storage maps" ); Ok(()) From 3cd457aa7de1f2f446b91c8a74308775ac30b2aa Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 9 Dec 2025 20:23:02 +0100 Subject: [PATCH 29/95] better --- crates/proto/src/domain/account.rs | 3 +- crates/store/src/state.rs | 79 ++++-------------------------- 2 files changed, 11 insertions(+), 71 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 5215d53669..607e8f9ab9 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -357,7 +357,8 @@ impl From for proto::account::AccountStorageHeader { /// and consume quite a bit of bandwidth, besides requiring additional memory on /// possibly low powered clients. /// -/// Hence `too_many_assets` is returned, which is indicating to the client to use the dedicated `SyncAccountVault` RPC endpoint and do incremental retrieval +/// Hence `too_many_assets` is returned, which is indicating to the client to use the dedicated +/// `SyncAccountVault` RPC endpoint and do incremental retrieval #[derive(Debug, Clone, PartialEq, Eq)] pub struct AccountVaultDetails { /// Flag indicating whether the vault has too many assets to return inline. diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 73d4e9735e..e6c3373202 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -527,18 +527,12 @@ impl State { } /// Updates storage map SMTs in the forest for changed accounts + #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = changed_account_ids.len()))] async fn update_storage_maps_in_forest( &self, changed_account_ids: &[AccountId], block_num: BlockNumber, ) -> Result<(), ApplyBlockError> { - tracing::debug!( - target: COMPONENT, - %block_num, - num_accounts = changed_account_ids.len(), - "Updating storage maps in forest" - ); - // Step 1: Query storage from database let account_storages = self.query_account_storages_from_db(changed_account_ids, block_num).await?; @@ -549,16 +543,11 @@ impl State { // Step 3: Update the forest with new SMTs self.populate_forest_with_storage_maps(map_slots_to_populate, block_num).await?; - tracing::info!( - target: COMPONENT, - %block_num, - "Successfully completed storage map SMT updates" - ); - Ok(()) } /// Queries account storage data from the database for the given accounts at a specific block + #[instrument(target = COMPONENT, skip_all, fields(num_accounts = account_ids.len()))] async fn query_account_storages_from_db( &self, account_ids: &[AccountId], @@ -571,16 +560,11 @@ impl State { account_storages.push((account_id, storage)); } - tracing::debug!( - target: COMPONENT, - num_accounts = account_storages.len(), - "Queried account storage from database" - ); - Ok(account_storages) } /// Extracts map-type storage slots and their entries from account storage data + #[instrument(target = COMPONENT, skip_all, fields(num_accounts = account_storages.len()))] fn extract_map_slots_from_storage( &self, account_storages: &[(AccountId, miden_objects::account::AccountStorage)], @@ -591,30 +575,17 @@ impl State { for (slot_idx, slot) in storage.slots().iter().enumerate() { if let StorageSlot::Map(storage_map) = slot { let entries = Vec::from_iter(storage_map.entries()); - - tracing::debug!( - target: COMPONENT, - %account_id, - slot_index = slot_idx, - num_entries = entries.len(), - "Extracted map slot entries" - ); - map_slots.push((*account_id, slot_idx as u8, entries)); } } } - tracing::debug!( - target: COMPONENT, - num_map_slots = map_slots.len(), - "Extracted all map slots from storage" - ); - + tracing::debug!(target: COMPONENT, num_map_slots = map_slots.len()); map_slots } /// Populates the forest with storage map SMTs for the given slots + #[instrument(target = COMPONENT, skip_all, fields(num_slots = map_slots.len()))] async fn populate_forest_with_storage_maps( &self, map_slots: Vec<(AccountId, u8, Vec<(&Word, &Word)>)>, @@ -646,59 +617,33 @@ impl State { forest_guard .storage_roots .insert((account_id, slot_idx, block_num), updated_root); - - tracing::debug!( - target: COMPONENT, - %account_id, - slot_index = slot_idx, - "Updated storage map SMT in forest" - ); } - tracing::info!( - target: COMPONENT, - total_tracked_roots = forest_guard.storage_roots.len(), - "Populated forest with storage maps" - ); - + tracing::debug!(target: COMPONENT, total_tracked_roots = forest_guard.storage_roots.len()); Ok(()) } /// Updates vault SMTs in the forest for changed accounts + #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = changed_account_ids.len()))] async fn update_vaults_in_forest( &self, changed_account_ids: &[AccountId], block_num: BlockNumber, ) -> Result<(), ApplyBlockError> { - tracing::debug!( - target: COMPONENT, - %block_num, - "Starting vault tracking" - ); - // Query vault assets for each updated account let mut vault_entries_to_populate = Vec::new(); for &account_id in changed_account_ids { let entries = self.db.select_account_vault_at_block(account_id, block_num).await?; - if entries.is_empty() { - tracing::debug!(%account_id, "Account has empty vault"); - } else { + if !entries.is_empty() { vault_entries_to_populate.push((account_id, entries)); } } if vault_entries_to_populate.is_empty() { - tracing::debug!("No vaults to populate"); return Ok(()); } - tracing::info!( - target: COMPONENT, - num_vaults = vault_entries_to_populate.len(), - "Queried vault assets" - ); - // Acquire a single write lock on the forest for the entire update operation. // Since apply_block() is already serialized by the `writer` Mutex, holding this lock // for the entire duration is acceptable and simplifies the code. @@ -723,13 +668,7 @@ impl State { forest_guard.vault_roots.insert((account_id, block_num), updated_root); } - tracing::info!( - target: COMPONENT, - %block_num, - total_vault_roots = forest_guard.vault_roots.len(), - "Successfully completed vault SMT updates" - ); - + tracing::debug!(target: COMPONENT, total_vault_roots = forest_guard.vault_roots.len()); Ok(()) } From c8f0eb1cc6984d641799337192a1e66f6594dec2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 9 Dec 2025 20:32:52 +0100 Subject: [PATCH 30/95] clippy et al --- .../store/src/db/models/queries/accounts.rs | 4 +- crates/store/src/db/tests.rs | 38 +++++++++---------- crates/store/src/state.rs | 11 +++--- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 87991ed261..313ca0c362 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -799,7 +799,7 @@ pub(crate) fn insert_account_vault_asset( /// Insert an account storage header into the DB using the given [`SqliteConnection`]. /// /// Sets `is_latest=true` for the new row and updates any existing -/// row with the same (account_id, slot_index) tuple to `is_latest=false`. +/// row with the same (`account_id`, `slot_index`) tuple to `is_latest=false`. /// /// # Returns /// @@ -1262,7 +1262,7 @@ fn compute_storage_commitment(slot_commitments: &[Word]) -> Word { Rpo256::hash_elements(&elements) } -/// Helper function to check if a block exists in the block_headers table. +/// Helper function to check if a block exists in the `block_headers` table. /// /// This should be called by all `_at_block` query functions to ensure that /// queries are only performed against blocks that have been produced. diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index dab35c41a9..12263c0f5b 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1416,6 +1416,25 @@ fn mock_block_account_update(account_id: AccountId, num: u64) -> BlockAccountUpd BlockAccountUpdate::new(account_id, num_to_word(num), AccountUpdateDetails::Private) } +// Helper function to create account with specific code for tests +fn create_account_with_code(code_str: &str, seed: [u8; 32]) -> Account { + let component_storage = + vec![StorageSlot::Value(Word::empty()), StorageSlot::Value(num_to_word(1))]; + + let component = + AccountComponent::compile(code_str, TransactionKernel::assembler(), component_storage) + .unwrap() + .with_supported_type(AccountType::RegularAccountUpdatableCode); + + AccountBuilder::new(seed) + .account_type(AccountType::RegularAccountUpdatableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap() +} + fn mock_block_transaction(account_id: AccountId, num: u64) -> TransactionHeader { let initial_state_commitment = Word::try_from([num, 0, 0, 0]).unwrap(); let final_account_commitment = Word::try_from([0, num, 0, 0]).unwrap(); @@ -1927,25 +1946,6 @@ fn test_select_account_code_at_block_with_updates() { create_block(&mut conn, block_num_2); create_block(&mut conn, block_num_3); - // Helper function to create account with specific code - fn create_account_with_code(code_str: &str, seed: [u8; 32]) -> Account { - let component_storage = - vec![StorageSlot::Value(Word::empty()), StorageSlot::Value(num_to_word(1))]; - - let component = - AccountComponent::compile(code_str, TransactionKernel::assembler(), component_storage) - .unwrap() - .with_supported_type(AccountType::RegularAccountUpdatableCode); - - AccountBuilder::new(seed) - .account_type(AccountType::RegularAccountUpdatableCode) - .storage_mode(AccountStorageMode::Public) - .with_component(component) - .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) - .build_existing() - .unwrap() - } - // Create initial account with code v1 at block 1 let code_v1_str = "\ export.account_procedure_1 diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index e6c3373202..75eb75582a 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -538,7 +538,7 @@ impl State { self.query_account_storages_from_db(changed_account_ids, block_num).await?; // Step 2: Extract map slots and their entries - let map_slots_to_populate = self.extract_map_slots_from_storage(&account_storages); + let map_slots_to_populate = Self::extract_map_slots_from_storage(&account_storages); // Step 3: Update the forest with new SMTs self.populate_forest_with_storage_maps(map_slots_to_populate, block_num).await?; @@ -565,10 +565,10 @@ impl State { /// Extracts map-type storage slots and their entries from account storage data #[instrument(target = COMPONENT, skip_all, fields(num_accounts = account_storages.len()))] - fn extract_map_slots_from_storage( - &self, - account_storages: &[(AccountId, miden_objects::account::AccountStorage)], - ) -> Vec<(AccountId, u8, Vec<(&Word, &Word)>)> { + #[allow(clippy::type_complexity)] + fn extract_map_slots_from_storage<'a>( + account_storages: &'a [(AccountId, miden_objects::account::AccountStorage)], + ) -> Vec<(AccountId, u8, Vec<(&'a Word, &'a Word)>)> { let mut map_slots = Vec::new(); for (account_id, storage) in account_storages { @@ -586,6 +586,7 @@ impl State { /// Populates the forest with storage map SMTs for the given slots #[instrument(target = COMPONENT, skip_all, fields(num_slots = map_slots.len()))] + #[allow(clippy::type_complexity)] async fn populate_forest_with_storage_maps( &self, map_slots: Vec<(AccountId, u8, Vec<(&Word, &Word)>)>, From ed7224e1de13c9d9c7c231a2694ff85891dcd165 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 10 Dec 2025 16:39:55 +0100 Subject: [PATCH 31/95] review --- crates/proto/src/domain/account.rs | 13 ------------- crates/store/src/db/mod.rs | 3 +-- crates/store/src/inner_forest.rs | 31 ++++++++++++++++++++++++++++++ crates/store/src/lib.rs | 1 + crates/store/src/state.rs | 27 +------------------------- 5 files changed, 34 insertions(+), 41 deletions(-) create mode 100644 crates/store/src/inner_forest.rs diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 607e8f9ab9..a9e84c6348 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -466,19 +466,6 @@ impl From for proto::rpc::AccountVaultDetails { /// - The `map_entries` field is empty (no data included) /// - Clients should use the dedicated `SyncStorageMaps` RPC endpoint /// - That endpoint supports pagination and block range filtering -/// -/// ## Future Enhancement (TODO) -/// -/// Currently, when `too_many_entries = true`, we return an empty list. A future improvement -/// would return a **partial SMT** with: -/// - A subset of entries (e.g., most frequently accessed) -/// - Merkle proofs for those entries -/// - Inner node commitments -/// -/// This would allow clients to verify partial data cryptographically while still -/// signaling that more data exists. The reason this matters: if all leaf values are -/// included, one can reconstruct the entire SMT; if even one is missing, one cannot. -/// By providing proofs, we enable verification of partial data. #[derive(Debug, Clone, PartialEq, Eq)] pub struct AccountStorageMapDetails { pub slot_index: u8, diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index a72d3e345d..5e32beafae 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -392,8 +392,7 @@ impl Db { .await } - /// Loads all the account commitments from the DB. - // TODO add a variant with block_num as arg + /// TODO marked for removal, replace with paged version #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_account_commitments(&self) -> Result> { self.transact("read all account commitments", move |conn| { diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs new file mode 100644 index 0000000000..a57db588e7 --- /dev/null +++ b/crates/store/src/inner_forest.rs @@ -0,0 +1,31 @@ +use std::collections::BTreeMap; + +use miden_objects::Word; +use miden_objects::account::AccountId; +use miden_objects::block::BlockNumber; +use miden_objects::crypto::merkle::SmtForest; + +/// Container for forest-related state that needs to be updated atomically. +pub(crate) struct InnerForest { + /// `SmtForest` for efficient account storage reconstruction. + /// Populated during block import with storage and vault SMTs. + pub(crate) storage_forest: SmtForest, + + /// Maps (`account_id`, `slot_index`, `block_num`) to SMT root. + /// Populated during block import for all storage map slots. + pub(crate) storage_roots: BTreeMap<(AccountId, u8, BlockNumber), Word>, + + /// Maps (`account_id`, `block_num`) to vault SMT root. + /// Tracks asset vault versions across all blocks with structural sharing. + pub(crate) vault_roots: BTreeMap<(AccountId, BlockNumber), Word>, +} + +impl InnerForest { + pub(crate) fn new() -> Self { + Self { + storage_forest: SmtForest::new(), + storage_roots: BTreeMap::new(), + vault_roots: BTreeMap::new(), + } + } +} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index a9974fa7ca..cee059e963 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -6,6 +6,7 @@ mod constants; mod db; mod errors; pub mod genesis; +mod inner_forest; mod server; pub mod state; diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 75eb75582a..1e32d95145 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -44,7 +44,6 @@ use miden_objects::crypto::merkle::{ MmrPeaks, MmrProof, PartialMmr, - SmtForest, SmtProof, SmtStorage, }; @@ -78,6 +77,7 @@ use crate::errors::{ StateInitializationError, StateSyncError, }; +use crate::inner_forest::InnerForest; use crate::{AccountTreeWithHistory, COMPONENT, DataDirectory}; // STRUCTURES @@ -91,31 +91,6 @@ pub struct TransactionInputs { pub new_account_id_prefix_is_unique: Option, } -/// Container for forest-related state that needs to be updated atomically. -struct InnerForest { - /// `SmtForest` for efficient account storage reconstruction. - /// Populated during block import with storage and vault SMTs. - storage_forest: SmtForest, - - /// Maps (`account_id`, `slot_index`, `block_num`) to SMT root. - /// Populated during block import for all storage map slots. - storage_roots: BTreeMap<(AccountId, u8, BlockNumber), Word>, - - /// Maps (`account_id`, `block_num`) to vault SMT root. - /// Tracks asset vault versions across all blocks with structural sharing. - vault_roots: BTreeMap<(AccountId, BlockNumber), Word>, -} - -impl InnerForest { - fn new() -> Self { - Self { - storage_forest: SmtForest::new(), - storage_roots: BTreeMap::new(), - vault_roots: BTreeMap::new(), - } - } -} - /// Container for state that needs to be updated atomically. struct InnerState where From 6336f41247a824cdfddd5eb2d488bb199716fc31 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 10 Dec 2025 20:12:08 +0100 Subject: [PATCH 32/95] address review comments --- .../db/migrations/2025062000000_setup/up.sql | 25 +- .../store/src/db/models/queries/accounts.rs | 444 ++++------------ .../src/db/models/queries/accounts/tests.rs | 478 ++++++++++++++++++ crates/store/src/db/schema.rs | 13 +- crates/store/src/db/tests.rs | 347 ------------- 5 files changed, 586 insertions(+), 721 deletions(-) create mode 100644 crates/store/src/db/models/queries/accounts/tests.rs diff --git a/crates/store/src/db/migrations/2025062000000_setup/up.sql b/crates/store/src/db/migrations/2025062000000_setup/up.sql index 0512499262..e02f23e0c8 100644 --- a/crates/store/src/db/migrations/2025062000000_setup/up.sql +++ b/crates/store/src/db/migrations/2025062000000_setup/up.sql @@ -19,14 +19,15 @@ CREATE TABLE accounts ( account_commitment BLOB NOT NULL, code_commitment BLOB, nonce INTEGER, + storage_header BLOB, -- Serialized AccountStorage from miden-objects is_latest BOOLEAN NOT NULL DEFAULT 0, -- Indicates if this is the latest state for this account_id PRIMARY KEY (account_id, block_num), CONSTRAINT all_null_or_none_null CHECK ( - (code_commitment IS NOT NULL AND nonce IS NOT NULL) + (code_commitment IS NOT NULL AND nonce IS NOT NULL AND storage_header IS NOT NULL) OR - (code_commitment IS NULL AND nonce IS NULL) + (code_commitment IS NULL AND nonce IS NULL AND storage_header IS NULL) ) ) WITHOUT ROWID; @@ -38,26 +39,6 @@ CREATE INDEX idx_accounts_block_num ON accounts(block_num); -- Index for joining with account_codes CREATE INDEX idx_accounts_code_commitment ON accounts(code_commitment) WHERE code_commitment IS NOT NULL; --- Table to store storage slot headers (slot types and commitments) -CREATE TABLE account_storage_headers ( - account_id BLOB NOT NULL, - block_num INTEGER NOT NULL, - slot_index INTEGER NOT NULL, - slot_type INTEGER NOT NULL, -- 0=Map, 1=Value (as per StorageSlotType) - slot_commitment BLOB NOT NULL, - is_latest BOOLEAN NOT NULL DEFAULT 0, - - PRIMARY KEY (account_id, block_num, slot_index), - CONSTRAINT slot_index_is_u8 CHECK (slot_index BETWEEN 0 AND 0xFF), - CONSTRAINT slot_type_in_enum CHECK (slot_type BETWEEN 0 AND 1), - FOREIGN KEY (account_id, block_num) REFERENCES accounts(account_id, block_num) ON DELETE CASCADE -) WITHOUT ROWID; - --- Index for joining with accounts table -CREATE INDEX idx_account_storage_headers_account_block ON account_storage_headers(account_id, block_num); --- Index for querying latest state -CREATE INDEX idx_account_storage_headers_latest ON account_storage_headers(account_id, is_latest) WHERE is_latest = 1; - CREATE TABLE notes ( committed_at INTEGER NOT NULL, -- Block number when the note was committed batch_index INTEGER NOT NULL, -- Index of batch in block, starting from 0 diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 313ca0c362..6ade45384b 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -28,9 +28,7 @@ use miden_objects::account::{ AccountId, AccountStorage, NonFungibleDeltaAction, - StorageMap, StorageSlot, - StorageSlotType, }; use miden_objects::asset::{Asset, AssetVault, AssetVaultKey, FungibleAsset}; use miden_objects::block::{BlockAccountUpdate, BlockNumber}; @@ -485,217 +483,52 @@ pub(crate) fn select_account_storage_map_values( Ok(StorageMapValuesPage { last_block_included, values }) } -/// Reconstruct a `StorageMap` from database entries using `SmtForest` -/// -/// This function builds an `SmtForest` from all key-value pairs at the specified block, -/// enabling efficient proof generation with structural sharing. The forest allows -/// maintaining multiple SMT versions in memory with shared nodes. -/// -/// # Arguments -/// -/// * `conn` - Database connection -/// * `account_id` - The account ID -/// * `block_num` - The block number -/// * `slot_index` - The storage slot index -/// -/// # Returns -/// -/// A reconstructed `StorageMap` backed by `SmtForest` with full proof capabilities. -pub(crate) fn reconstruct_storage_map_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, - slot_index: u8, -) -> Result { - use schema::account_storage_map_values as t; - - // Check if the requested block exists (returns error if not) - block_exists(conn, block_num)?; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - let slot_sql = slot_to_raw_sql(slot_index); - - // Query all entries for this slot at or before the given block - let raw: Vec<(Vec, Vec)> = SelectDsl::select(t::table, (t::key, t::value)) - .filter( - t::account_id - .eq(&account_id_bytes) - .and(t::slot.eq(slot_sql)) - .and(t::block_num.le(block_num_sql)), - ) - .load(conn)?; - - // Parse entries - let entries: Vec<(Word, Word)> = raw - .into_iter() - .map(|(k, v)| Ok((Word::read_from_bytes(&k)?, Word::read_from_bytes(&v)?))) - .collect::, DatabaseError>>()?; - - let entry_count = entries.len(); - - // StorageMap::with_entries internally uses an SMT which can be backed by SmtForest - // The SMT is built with structural sharing for memory efficiency - miden_objects::account::StorageMap::with_entries(entries).map_err(|e| { - DatabaseError::DataCorrupted(format!( - "Failed to create StorageMap from {entry_count} entries: {e}" - )) - }) -} - -/// Reconstruct `AccountStorage` from database tables for a specific account at a specific block -/// -/// This function queries the `account_storage_headers` table to get slot metadata and reconstructs -/// the `AccountStorage` without deserializing a blob. For Map slots, we only store the commitment -/// since the actual map data is in `account_storage_map_values`. -/// -/// # Returns -/// -/// The reconstructed `AccountStorage`, or an error if reconstruction fails. +/// Returns account storage at a given block by deserializing the storage header blob. +/// Returns account storage at a given block by reading from `accounts.storage_header`. pub(crate) fn select_account_storage_at_block( conn: &mut SqliteConnection, account_id: AccountId, block_num: BlockNumber, ) -> Result { - use schema::account_storage_headers as t; - - // Check if the requested block exists (returns error if not) block_exists(conn, block_num)?; - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - - // Query storage headers for this account at this block - let headers: Vec = - SelectDsl::select(t::table, AccountStorageHeaderRaw::as_select()) - .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.eq(block_num_sql))) - .order(t::slot_index.asc()) - .load(conn)?; - - if headers.is_empty() { - // No storage headers means empty storage - return Ok(AccountStorage::new(Vec::new())?); - } - - // Build slots from headers - let mut slots = Vec::with_capacity(headers.len()); - - for header in headers { - let slot_type = StorageSlotType::from_raw_sql(header.slot_type)?; - - let commitment = Word::read_from_bytes(&header.slot_commitment)?; - - let slot = match slot_type { - StorageSlotType::Map => { - // For Map slots, we create an empty map - // The actual map data is queried separately when needed from - // account_storage_map_values - - // Create an empty storage map - let storage_map = StorageMap::new(); - StorageSlot::Map(storage_map) - }, - StorageSlotType::Value => { - // For Value slots, the commitment IS the value - StorageSlot::Value(commitment) - }, - }; - - slots.push(slot); - } - - Ok(AccountStorage::new(slots)?) -} - -/// Select account storage headers at a specific block (lightweight query). -/// -/// Returns tuples of `(slot_index, slot_type, commitment)` without reconstructing full slots. -#[allow(dead_code)] // Helper for future SmtForest integration -pub(crate) fn select_account_storage_headers_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result, DatabaseError> { - use schema::account_storage_headers as t; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - - let headers: Vec = - SelectDsl::select(t::table, AccountStorageHeaderRaw::as_select()) - .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.le(block_num_sql))) - .order(t::slot_index.asc()) - .load(conn)?; - - headers - .into_iter() - .map(|h| { - let slot_index = raw_sql_to_slot(h.slot_index); - let slot_type = StorageSlotType::from_raw_sql(h.slot_type)?; - let commitment = Word::read_from_bytes(&h.slot_commitment)?; - Ok((slot_index, slot_type, commitment)) - }) - .collect() + let storage_header_blob: Option> = + SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) + .filter( + schema::accounts::account_id + .eq(account_id.to_bytes()) + .and(schema::accounts::block_num.le(block_num.to_raw_sql())), + ) + .order(schema::accounts::block_num.desc()) + .limit(1) + .first(conn) + .optional()? + .flatten(); + + storage_header_blob + .map(|blob| AccountStorage::read_from_bytes(&blob).map_err(Into::into)) + .unwrap_or_else(|| Ok(AccountStorage::new(Vec::new())?)) } -/// Reconstruct `AccountStorage` from the latest state in the database -/// -/// This queries only the latest storage headers (where `is_latest=true`) for faster reconstruction -/// Select the latest storage headers for an account -/// -/// This function queries the `account_storage_headers` table for the latest state of an account's -/// storage slots, using the `is_latest=true` flag for efficiency. -/// -/// # Returns -/// -/// The reconstructed `AccountStorage` from the latest storage headers. +/// Select latest account storage by querying `accounts` where `is_latest=true`. pub(crate) fn select_latest_account_storage( conn: &mut SqliteConnection, account_id: AccountId, ) -> Result { - use schema::account_storage_headers as t; - - let account_id_bytes = account_id.to_bytes(); - - // Query latest storage headers for this account - let headers: Vec = - SelectDsl::select(t::table, AccountStorageHeaderRaw::as_select()) - .filter(t::account_id.eq(&account_id_bytes).and(t::is_latest.eq(true))) - .order(t::slot_index.asc()) - .load(conn)?; - - if headers.is_empty() { - // No storage headers means empty storage - return Ok(AccountStorage::new(Vec::new())?); - } - - // Build slots from headers - let mut slots = Vec::with_capacity(headers.len()); - - for header in headers { - let slot_type = StorageSlotType::from_raw_sql(header.slot_type)?; - let slot_index = raw_sql_to_slot(header.slot_index); - let block_num = BlockNumber::from_raw_sql(header.block_num)?; - let commitment = Word::read_from_bytes(&header.slot_commitment)?; - - let slot = match slot_type { - StorageSlotType::Map => { - // For Map slots, reconstruct the full SMT from database entries - // This allows serving proofs for any key in the map - let storage_map = - reconstruct_storage_map_at_block(conn, account_id, block_num, slot_index)?; - StorageSlot::Map(storage_map) - }, - StorageSlotType::Value => { - // For Value slots, the commitment IS the value - StorageSlot::Value(commitment) - }, - }; - - slots.push(slot); - } + let storage_header_blob: Option> = + SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) + .filter( + schema::accounts::account_id + .eq(account_id.to_bytes()) + .and(schema::accounts::is_latest.eq(true)), + ) + .first(conn) + .optional()? + .flatten(); - Ok(AccountStorage::new(slots)?) + storage_header_blob + .map(|blob| AccountStorage::read_from_bytes(&blob).map_err(Into::into)) + .unwrap_or_else(|| Ok(AccountStorage::new(Vec::new())?)) } #[derive(Queryable, Selectable)] @@ -719,19 +552,6 @@ impl TryFrom for AccountVaultValue { } } -#[derive(Debug, Clone, Queryable, Selectable)] -#[diesel(table_name = schema::account_storage_headers)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -#[allow(dead_code)] // Fields used by Diesel, not directly in Rust code -pub struct AccountStorageHeaderRaw { - pub account_id: Vec, - pub block_num: i64, - pub slot_index: i32, - pub slot_type: i32, - pub slot_commitment: Vec, - pub is_latest: bool, -} - #[derive(Debug, Clone, PartialEq, Eq, Selectable, Queryable, QueryableByName)] #[diesel(table_name = schema::accounts)] #[diesel(check_for_backend(Sqlite))] @@ -796,59 +616,6 @@ pub(crate) fn insert_account_vault_asset( }) } -/// Insert an account storage header into the DB using the given [`SqliteConnection`]. -/// -/// Sets `is_latest=true` for the new row and updates any existing -/// row with the same (`account_id`, `slot_index`) tuple to `is_latest=false`. -/// -/// # Returns -/// -/// The number of affected rows. -#[cfg(test)] -pub(crate) fn insert_account_storage_header( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, - slot_index: u8, - slot_type: StorageSlotType, - slot_commitment: Word, -) -> Result { - use schema::account_storage_headers as t; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - let slot_index_sql = slot_to_raw_sql(slot_index); - let slot_type_sql = slot_type.to_raw_sql(); - let slot_commitment_bytes = slot_commitment.to_bytes(); - - diesel::Connection::transaction(conn, |conn| { - // Update existing headers for this slot to set is_latest=false - let update_count = diesel::update(t::table) - .filter( - t::account_id - .eq(&account_id_bytes) - .and(t::slot_index.eq(slot_index_sql)) - .and(t::is_latest.eq(true)), - ) - .set(t::is_latest.eq(false)) - .execute(conn)?; - - // Insert the new latest row - let insert_count = diesel::insert_into(t::table) - .values(( - t::account_id.eq(&account_id_bytes), - t::block_num.eq(block_num_sql), - t::slot_index.eq(slot_index_sql), - t::slot_type.eq(slot_type_sql), - t::slot_commitment.eq(&slot_commitment_bytes), - t::is_latest.eq(true), - )) - .execute(conn)?; - - Ok(update_count + insert_count) - }) -} - /// Insert an account storage map value into the DB using the given [`SqliteConnection`]. /// /// Sets `is_latest=true` for the new row and updates any existing @@ -1090,6 +857,7 @@ pub(crate) fn upsert_accounts( code_commitment: full_account .as_ref() .map(|account| account.code().commitment().to_bytes()), + storage_header: full_account.as_ref().map(|account| account.storage().to_bytes()), is_latest: true, }; @@ -1097,12 +865,10 @@ pub(crate) fn upsert_accounts( .values(&account_value) .execute(conn)?; - // insert pending storage map entries for (acc_id, slot, key, value) in pending_storage_inserts { insert_account_storage_map_value(conn, acc_id, block_num, slot, key, value)?; } - // insert pending vault-asset entries for (acc_id, vault_key, update) in pending_asset_inserts { insert_account_vault_asset(conn, acc_id, block_num, vault_key, update)?; } @@ -1139,17 +905,6 @@ pub(crate) struct AccountCodeRowInsert { pub(crate) code: Vec, } -#[derive(Insertable, AsChangeset, Debug, Clone)] -#[diesel(table_name = schema::account_storage_headers)] -pub(crate) struct AccountStorageHeaderInsert { - pub(crate) account_id: Vec, - pub(crate) block_num: i64, - pub(crate) slot_index: i32, - pub(crate) slot_type: i32, - pub(crate) slot_commitment: Vec, - pub(crate) is_latest: bool, -} - #[derive(Insertable, AsChangeset, Debug, Clone)] #[diesel(table_name = schema::accounts)] pub(crate) struct AccountRowInsert { @@ -1159,6 +914,7 @@ pub(crate) struct AccountRowInsert { pub(crate) account_commitment: Vec, pub(crate) code_commitment: Option>, pub(crate) nonce: Option, + pub(crate) storage_header: Option>, pub(crate) is_latest: bool, } @@ -1206,7 +962,7 @@ pub(crate) struct AccountStorageMapRowInsert { pub(crate) is_latest: bool, } -/// Queries vault assets (key, value) pairs at a specific block +/// Query vault assets at a specific block by finding the most recent update for each `vault_key`. pub(crate) fn select_account_vault_at_block( conn: &mut SqliteConnection, account_id: AccountId, @@ -1218,48 +974,54 @@ pub(crate) fn select_account_vault_at_block( block_exists(conn, block_num)?; let account_id_bytes = account_id.to_bytes(); - let block_num_sql = i64::from(block_num.as_u32()); - let raw: Vec<(Vec, Option>)> = SelectDsl::select( + let block_num_sql = block_num.to_raw_sql(); + + // Since Diesel doesn't support composite keys in subqueries easily, we use a two-step approach: + // Step 1: Get max block_num for each vault_key + let max_blocks: Vec<(Vec, i64)> = QueryDsl::select( t::table .filter(t::account_id.eq(&account_id_bytes)) .filter(t::block_num.le(block_num_sql)) - .order(t::block_num.desc()) - .limit(1), - (t::vault_key, t::asset), + .group_by(t::vault_key), + (t::vault_key, diesel::dsl::max(t::block_num)), ) - .load(conn)?; + .load::<(Vec, Option)>(conn)? + .into_iter() + .filter_map(|(key, maybe_block)| maybe_block.map(|block| (key, block))) + .collect(); - let entries = raw - .into_iter() - .filter_map(|(key_bytes, maybe_asset_bytes)| { - let key = Word::read_from_bytes(&key_bytes).ok()?; - let asset_bytes = maybe_asset_bytes?; - let value = Word::read_from_bytes(&asset_bytes).ok()?; - Some((key, value)) - }) - .collect(); - - Ok(entries) -} + if max_blocks.is_empty() { + return Ok(Vec::new()); + } -/// Computes the storage commitment from a list of slot commitments. -/// -/// This replicates the logic from `AccountStorage::commitment()` which hashes all slot -/// commitments together. -/// -/// # Arguments -/// -/// * `slot_commitments` - Vector of slot commitment words -/// -/// # Returns -/// -/// The storage commitment as a `Word` -fn compute_storage_commitment(slot_commitments: &[Word]) -> Word { - use miden_objects::crypto::hash::rpo::Rpo256; + // Step 2: Fetch the full rows matching (vault_key, block_num) pairs + let mut entries = Vec::new(); + for (vault_key_bytes, max_block) in max_blocks { + let result: Option<(Vec, Option>)> = QueryDsl::select( + t::table.filter( + t::account_id + .eq(&account_id_bytes) + .and(t::vault_key.eq(&vault_key_bytes)) + .and(t::block_num.eq(max_block)), + ), + (t::vault_key, t::asset), + ) + .first(conn) + .optional()?; + + if let Some((key_bytes, Some(asset_bytes))) = result { + if let (Ok(key), Ok(value)) = + (Word::read_from_bytes(&key_bytes), Word::read_from_bytes(&asset_bytes)) + { + entries.push((key, value)); + } + } + } - let elements: Vec = slot_commitments.iter().flat_map(|w| w.iter()).copied().collect(); + // Sort by vault_key for consistent ordering + entries.sort_by_key(|(key, _)| *key); - Rpo256::hash_elements(&elements) + Ok(entries) } /// Helper function to check if a block exists in the `block_headers` table. @@ -1339,12 +1101,18 @@ pub(crate) fn select_account_code_at_block( Ok(result) } +#[derive(Debug, Clone, Queryable)] +struct AccountHeaderDataRaw { + code_commitment: Option>, + nonce: Option, + storage_header: Option>, +} + /// Queries the account header for a specific account at a specific block number. /// /// This reconstructs the `AccountHeader` by joining multiple tables: -/// - `accounts` table for `account_id`, `nonce`, `code_commitment` +/// - `accounts` table for `account_id`, `nonce`, `code_commitment`, `storage_header` /// - `account_vault_headers` table for `vault_root` -/// - `account_storage_headers` table for storage slot commitments (to compute `storage_commitment`) /// /// Returns `None` if the account doesn't exist at that block. /// @@ -1364,27 +1132,33 @@ pub(crate) fn select_account_header_at_block( account_id: AccountId, block_num: BlockNumber, ) -> Result, DatabaseError> { - use schema::{account_storage_headers, account_vault_headers, accounts}; + use schema::{account_vault_headers, accounts}; - // Check if the requested block exists (returns error if not) block_exists(conn, block_num)?; let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); - let account_data: Option<(Option>, Option)> = SelectDsl::select( + + let account_data: Option = SelectDsl::select( accounts::table .filter(accounts::account_id.eq(&account_id_bytes)) .filter(accounts::block_num.le(block_num_sql)) .order(accounts::block_num.desc()) .limit(1), - (accounts::code_commitment, accounts::nonce), + (accounts::code_commitment, accounts::nonce, accounts::storage_header), ) .first(conn) .optional()?; - let Some((code_commitment_bytes, nonce_raw)) = account_data else { + let Some(AccountHeaderDataRaw { + code_commitment: code_commitment_bytes, + nonce: nonce_raw, + storage_header: storage_header_blob, + }) = account_data + else { return Ok(None); }; + let vault_root_bytes: Option> = SelectDsl::select( account_vault_headers::table .filter(account_vault_headers::account_id.eq(&account_id_bytes)) @@ -1396,26 +1170,13 @@ pub(crate) fn select_account_header_at_block( .first(conn) .optional()?; - let storage_slots: Vec<(i32, i32, Vec)> = SelectDsl::select( - account_storage_headers::table - .filter(account_storage_headers::account_id.eq(&account_id_bytes)) - .filter(account_storage_headers::block_num.le(block_num_sql)) - .order(account_storage_headers::block_num.desc()) - .limit(1), - ( - account_storage_headers::slot_index, - account_storage_headers::slot_type, - account_storage_headers::slot_commitment, - ), - ) - .load(conn)?; - - let slot_commitments: Vec = storage_slots - .into_iter() - .map(|(_slot_index, _slot_type, commitment_bytes)| Word::read_from_bytes(&commitment_bytes)) - .collect::, _>>()?; - - let storage_commitment = compute_storage_commitment(&slot_commitments); + let storage_commitment = match storage_header_blob { + Some(blob) => { + let storage = AccountStorage::read_from_bytes(&blob)?; + storage.commitment() + }, + None => Word::default(), + }; let code_commitment = code_commitment_bytes .map(|bytes| Word::read_from_bytes(&bytes)) @@ -1437,3 +1198,6 @@ pub(crate) fn select_account_header_at_block( code_commitment, ))) } + +#[cfg(test)] +mod tests; diff --git a/crates/store/src/db/models/queries/accounts/tests.rs b/crates/store/src/db/models/queries/accounts/tests.rs new file mode 100644 index 0000000000..b68df7367e --- /dev/null +++ b/crates/store/src/db/models/queries/accounts/tests.rs @@ -0,0 +1,478 @@ +use assert_matches::assert_matches; +use diesel::{Connection, RunQueryDsl}; +use diesel_migrations::MigrationHarness; +use miden_lib::account::auth::AuthRpoFalcon512; +use miden_lib::transaction::TransactionKernel; +use miden_node_utils::fee::test_fee_params; +use miden_objects::account::auth::PublicKeyCommitment; +use miden_objects::account::{ + AccountBuilder, + AccountComponent, + AccountIdVersion, + AccountStorageMode, + AccountType, + StorageSlot, +}; +use miden_objects::{EMPTY_WORD, Word}; + +use super::*; +use crate::db::migrations::MIGRATIONS; + +fn setup_test_db() -> SqliteConnection { + let mut conn = + SqliteConnection::establish(":memory:").expect("Failed to create in-memory database"); + + conn.run_pending_migrations(MIGRATIONS).expect("Failed to run migrations"); + + conn +} + +fn create_test_account_with_storage() -> (Account, AccountId) { + // Create a simple public account with one value storage slot + let account_id = AccountId::dummy( + [1u8; 15], + AccountIdVersion::Version0, + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + ); + + let storage_value = Word::from([Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]); + let component_storage = vec![StorageSlot::Value(storage_value)]; + + let component = AccountComponent::compile( + "export.foo push.1 end", + TransactionKernel::assembler(), + component_storage, + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); + + let account = AccountBuilder::new([1u8; 32]) + .account_type(AccountType::RegularAccountImmutableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + (account, account_id) +} + +fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { + use miden_objects::block::BlockHeader; + + use crate::db::schema::block_headers; + + let block_header = BlockHeader::new( + 1_u8.into(), + Word::default(), + block_num, + Word::default(), + Word::default(), + Word::default(), + Word::default(), + Word::default(), + Word::default(), + Word::default(), + test_fee_params(), + 0_u8.into(), + ); + + diesel::insert_into(block_headers::table) + .values(( + block_headers::block_num.eq(i64::from(block_num.as_u32())), + block_headers::block_header.eq(block_header.to_bytes()), + )) + .execute(conn) + .expect("Failed to insert block header"); +} + +#[test] +fn test_upsert_accounts_inserts_storage_header() { + let mut conn = setup_test_db(); + let (account, account_id) = create_test_account_with_storage(); + + // Block 1 + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + let storage_commitment_original = account.storage().commitment(); + let storage_slots_len = account.storage().slots().len(); + let account_commitment = account.commitment(); + + // Create full state delta from the account + let delta = AccountDelta::try_from(account).unwrap(); + assert!(delta.is_full_state(), "Delta should be full state"); + + let account_update = + BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Delta(delta)); + + // Upsert account + let result = upsert_accounts(&mut conn, &[account_update], block_num); + assert!(result.is_ok(), "upsert_accounts failed: {:?}", result.err()); + assert_eq!(result.unwrap(), 1, "Expected 1 account to be inserted"); + + // Query storage header back + let queried_storage = select_latest_account_storage(&mut conn, account_id) + .expect("Failed to query storage header"); + + // Verify storage commitment matches + assert_eq!( + queried_storage.commitment(), + storage_commitment_original, + "Storage commitment mismatch" + ); + + // Verify number of slots matches + assert_eq!(queried_storage.slots().len(), storage_slots_len, "Storage slots count mismatch"); + + // Verify exactly 1 latest account with storage exists + let header_count: i64 = schema::accounts::table + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .filter(schema::accounts::storage_header.is_not_null()) + .count() + .get_result(&mut conn) + .expect("Failed to count accounts with storage"); + + assert_eq!(header_count, 1, "Expected exactly 1 latest account with storage"); +} + +#[test] +fn test_upsert_accounts_updates_is_latest_flag() { + let mut conn = setup_test_db(); + let (account, account_id) = create_test_account_with_storage(); + + // Block 1 and 2 + let block_num_1 = BlockNumber::from_epoch(0); + let block_num_2 = BlockNumber::from_epoch(1); + + insert_block_header(&mut conn, block_num_1); + insert_block_header(&mut conn, block_num_2); + + // Save storage commitment before moving account + let storage_commitment_1 = account.storage().commitment(); + let account_commitment_1 = account.commitment(); + + // First update with original account - full state delta + let delta_1 = AccountDelta::try_from(account).unwrap(); + + let account_update_1 = BlockAccountUpdate::new( + account_id, + account_commitment_1, + AccountUpdateDetails::Delta(delta_1), + ); + + upsert_accounts(&mut conn, &[account_update_1], block_num_1).expect("First upsert failed"); + + // Create modified account with different storage value + let storage_value_modified = + Word::from([Felt::new(10), Felt::new(20), Felt::new(30), Felt::new(40)]); + let component_storage_modified = vec![StorageSlot::Value(storage_value_modified)]; + + let component_2 = AccountComponent::compile( + "export.foo push.1 end", + TransactionKernel::assembler(), + component_storage_modified, + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); + + let account_2 = AccountBuilder::new([1u8; 32]) + .account_type(AccountType::RegularAccountImmutableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component_2) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + let storage_commitment_2 = account_2.storage().commitment(); + let account_commitment_2 = account_2.commitment(); + + // Second update with modified account - full state delta + let delta_2 = AccountDelta::try_from(account_2).unwrap(); + + let account_update_2 = BlockAccountUpdate::new( + account_id, + account_commitment_2, + AccountUpdateDetails::Delta(delta_2), + ); + + upsert_accounts(&mut conn, &[account_update_2], block_num_2).expect("Second upsert failed"); + + // Verify 2 total account rows exist (both historical records) + let total_accounts: i64 = schema::accounts::table + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .count() + .get_result(&mut conn) + .expect("Failed to count total accounts"); + + assert_eq!(total_accounts, 2, "Expected 2 total account records"); + + // Verify only 1 is marked as latest + let latest_accounts: i64 = schema::accounts::table + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .count() + .get_result(&mut conn) + .expect("Failed to count latest accounts"); + + assert_eq!(latest_accounts, 1, "Expected exactly 1 latest account"); + + // Verify latest storage matches second update + let latest_storage = select_latest_account_storage(&mut conn, account_id) + .expect("Failed to query latest storage"); + + assert_eq!( + latest_storage.commitment(), + storage_commitment_2, + "Latest storage should match second update" + ); + + // Verify historical query returns first update + let storage_at_block_1 = select_account_storage_at_block(&mut conn, account_id, block_num_1) + .expect("Failed to query storage at block 1"); + + assert_eq!( + storage_at_block_1.commitment(), + storage_commitment_1, + "Storage at block 1 should match first update" + ); +} + +#[test] +fn test_upsert_accounts_with_incremental_delta() { + use std::collections::BTreeMap; + + use miden_objects::account::delta::{AccountStorageDelta, AccountVaultDelta}; + + let mut conn = setup_test_db(); + let (account, account_id) = create_test_account_with_storage(); + + let block_num_1 = BlockNumber::from_epoch(0); + let block_num_2 = BlockNumber::from_epoch(1); + + insert_block_header(&mut conn, block_num_1); + insert_block_header(&mut conn, block_num_2); + + // First update with full state + let storage_commitment_1 = account.storage().commitment(); + let account_commitment_1 = account.commitment(); + let nonce_1 = account.nonce(); + let delta_1 = AccountDelta::try_from(account).unwrap(); + + let account_update_1 = BlockAccountUpdate::new( + account_id, + account_commitment_1, + AccountUpdateDetails::Delta(delta_1), + ); + + upsert_accounts(&mut conn, &[account_update_1], block_num_1).expect("First upsert failed"); + + // Create incremental delta (only modify storage value slot 1) + let new_storage_value = + Word::from([Felt::new(100), Felt::new(200), Felt::new(300), Felt::new(400)]); + + let mut storage_delta_values = BTreeMap::new(); + storage_delta_values.insert(1u8, new_storage_value); // Update slot 1 (component storage) + + let storage_delta = AccountStorageDelta::from_parts(storage_delta_values, BTreeMap::new()) + .expect("Failed to create storage delta"); + let incremental_delta = + AccountDelta::new(account_id, storage_delta, AccountVaultDelta::default(), nonce_1) + .expect("Failed to create incremental delta"); + + // Reconstruct expected account after delta + let account_after = reconstruct_full_account_from_db(&mut conn, account_id) + .expect("Failed to reconstruct account"); + let mut expected_account = account_after.clone(); + expected_account + .apply_delta(&incremental_delta) + .expect("Failed to apply delta to expected account"); + + let storage_commitment_2 = expected_account.storage().commitment(); + let account_commitment_2 = expected_account.commitment(); + + let account_update_2 = BlockAccountUpdate::new( + account_id, + account_commitment_2, + AccountUpdateDetails::Delta(incremental_delta), + ); + + upsert_accounts(&mut conn, &[account_update_2], block_num_2) + .expect("Second upsert with incremental delta failed"); + + // Verify latest storage matches expected state + let latest_storage = select_latest_account_storage(&mut conn, account_id) + .expect("Failed to query latest storage"); + + assert_eq!( + latest_storage.commitment(), + storage_commitment_2, + "Storage commitment should match after incremental delta" + ); + + // Verify historical storage is preserved + let storage_at_block_1 = select_account_storage_at_block(&mut conn, account_id, block_num_1) + .expect("Failed to query storage at block 1"); + + assert_eq!( + storage_at_block_1.commitment(), + storage_commitment_1, + "Historical storage should be unchanged" + ); +} + +#[test] +fn test_upsert_accounts_with_multiple_storage_slots() { + let mut conn = setup_test_db(); + + // Create account with 3 storage slots + let account_id = AccountId::dummy( + [2u8; 15], + AccountIdVersion::Version0, + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + ); + + let slot_value_1 = Word::from([Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]); + let slot_value_2 = Word::from([Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]); + let slot_value_3 = Word::from([Felt::new(9), Felt::new(10), Felt::new(11), Felt::new(12)]); + + let component_storage = vec![ + StorageSlot::Value(slot_value_1), + StorageSlot::Value(slot_value_2), + StorageSlot::Value(slot_value_3), + ]; + + let component = AccountComponent::compile( + "export.foo push.1 end", + TransactionKernel::assembler(), + component_storage, + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); + + let account = AccountBuilder::new([2u8; 32]) + .account_type(AccountType::RegularAccountImmutableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + let storage_commitment = account.storage().commitment(); + let account_commitment = account.commitment(); + let delta = AccountDelta::try_from(account).unwrap(); + + let account_update = + BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Delta(delta)); + + upsert_accounts(&mut conn, &[account_update], block_num) + .expect("Upsert with multiple storage slots failed"); + + // Query back and verify + let queried_storage = + select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + + assert_eq!(queried_storage.commitment(), storage_commitment, "Storage commitment mismatch"); + + // Note: Auth component adds 1 storage slot, so 3 component slots + 1 auth = 4 total + assert_eq!( + queried_storage.slots().len(), + 4, + "Expected 4 storage slots (3 component + 1 auth)" + ); + + // Verify individual slot values (skipping auth slot at index 0) + assert_matches!( + queried_storage.slots().get(1).expect("Slot 1 should exist"), + &StorageSlot::Value(v) if v == slot_value_1, + "Slot 1 value mismatch" + ); + assert_matches!( + queried_storage.slots().get(2).expect("Slot 2 should exist"), + &StorageSlot::Value(v) if v == slot_value_2, + "Slot 2 value mismatch" + ); + assert_matches!( + queried_storage.slots().get(3).expect("Slot 3 should exist"), + &StorageSlot::Value(v) if v == slot_value_3, + "Slot 3 value mismatch" + ); +} + +#[test] +fn test_upsert_accounts_with_empty_storage() { + let mut conn = setup_test_db(); + + // Create account with no storage slots + let account_id = AccountId::dummy( + [3u8; 15], + AccountIdVersion::Version0, + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + ); + + let component = AccountComponent::compile( + "export.foo push.1 end", + TransactionKernel::assembler(), + vec![], // Empty storage + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); + + let account = AccountBuilder::new([3u8; 32]) + .account_type(AccountType::RegularAccountImmutableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + let storage_commitment = account.storage().commitment(); + let account_commitment = account.commitment(); + let delta = AccountDelta::try_from(account).unwrap(); + + let account_update = + BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Delta(delta)); + + upsert_accounts(&mut conn, &[account_update], block_num) + .expect("Upsert with empty storage failed"); + + // Query back and verify + let queried_storage = + select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + + assert_eq!( + queried_storage.commitment(), + storage_commitment, + "Storage commitment mismatch for empty storage" + ); + + // Note: Auth component adds 1 storage slot, so even "empty" accounts have 1 slot + assert_eq!(queried_storage.slots().len(), 1, "Expected 1 storage slot (auth component)"); + + // Verify the storage header blob exists in database + let storage_header_exists: Option = SelectDsl::select( + schema::accounts::table + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)), + schema::accounts::storage_header.is_not_null(), + ) + .first(&mut conn) + .optional() + .expect("Failed to check storage header existence"); + + assert_eq!( + storage_header_exists, + Some(true), + "Storage header blob should exist even for empty storage" + ); +} diff --git a/crates/store/src/db/schema.rs b/crates/store/src/db/schema.rs index 4929d3e106..18d557bdde 100644 --- a/crates/store/src/db/schema.rs +++ b/crates/store/src/db/schema.rs @@ -1,16 +1,5 @@ // @generated automatically by Diesel CLI. -diesel::table! { - account_storage_headers (account_id, block_num, slot_index) { - account_id -> Binary, - block_num -> BigInt, - slot_index -> Integer, - slot_type -> Integer, - slot_commitment -> Binary, - is_latest -> Bool, - } -} - diesel::table! { account_storage_map_values (account_id, block_num, slot, key) { account_id -> Binary, @@ -48,6 +37,7 @@ diesel::table! { account_commitment -> Binary, code_commitment -> Nullable, nonce -> Nullable, + storage_header -> Nullable, block_num -> BigInt, is_latest -> Bool, } @@ -130,7 +120,6 @@ diesel::joinable!(transactions -> block_headers (block_num)); diesel::allow_tables_to_appear_in_same_query!( account_codes, - account_storage_headers, account_storage_map_values, accounts, account_vault_assets, diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 12263c0f5b..e4910b1a43 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -20,14 +20,11 @@ use miden_objects::account::{ AccountDelta, AccountId, AccountIdVersion, - AccountStorage, AccountStorageDelta, AccountStorageMode, AccountType, AccountVaultDelta, - StorageMap, StorageSlot, - StorageSlotType, }; use miden_objects::asset::{Asset, AssetVaultKey, FungibleAsset}; use miden_objects::block::{ @@ -1541,350 +1538,6 @@ fn mock_account_code_and_storage( // STORAGE RECONSTRUCTION TESTS // ================================================================================================ -#[test] -#[miden_node_test_macro::enable_logging] -fn test_storage_reconstruction_latest_state() { - let mut conn = create_db(); - - // Create an account with storage slots - let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - let block_num = BlockNumber::from(1); - - // Create the block before inserting storage data - create_block(&mut conn, block_num); - - // Create test storage with Value and Map slots - let value_slot = StorageSlot::Value(num_to_word(42)); - let mut storage_map = StorageMap::new(); - let _ = storage_map.insert(num_to_word(1), num_to_word(100)); - let _ = storage_map.insert(num_to_word(2), num_to_word(200)); - let map_slot = StorageSlot::Map(storage_map.clone()); - - let _storage = AccountStorage::new(vec![value_slot, map_slot]).unwrap(); - - // Insert storage headers for both slots - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num, - 0, // slot_index - miden_objects::account::StorageSlotType::Value, - num_to_word(42), - ) - .unwrap(); - - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num, - 1, // slot_index - miden_objects::account::StorageSlotType::Map, - storage_map.root(), - ) - .unwrap(); - - // Insert map values - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block_num, - 1, // slot - num_to_word(1), // key - num_to_word(100), // value - ) - .unwrap(); - - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block_num, - 1, // slot - num_to_word(2), // key - num_to_word(200), // value - ) - .unwrap(); - - // Reconstruct storage from latest state - let reconstructed_storage = - queries::select_latest_account_storage(&mut conn, account_id).unwrap(); - - // Verify reconstructed storage - assert_eq!(reconstructed_storage.slots().len(), 2); - - // Check Value slot - match &reconstructed_storage.slots()[0] { - StorageSlot::Value(v) => assert_eq!(*v, num_to_word(42)), - StorageSlot::Map(_) => panic!("Expected Value slot"), - } - - // Check Map slot (commitment should match) - match &reconstructed_storage.slots()[1] { - StorageSlot::Map(_) => { - // The map should be reconstructed (empty but with correct slot type) - // Actual values would need to be queried separately from account_storage_map_values - }, - StorageSlot::Value(_) => panic!("Expected Map slot"), - } -} - -#[test] -#[miden_node_test_macro::enable_logging] -fn test_storage_reconstruction_historical_state() { - let mut conn = create_db(); - - let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - - // Block 1: Initial storage - let block_num_1 = BlockNumber::from(1); - create_block(&mut conn, block_num_1); - - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num_1, - 0, - miden_objects::account::StorageSlotType::Value, - num_to_word(10), - ) - .unwrap(); - - // Block 2: Updated storage - let block_num_2 = BlockNumber::from(2); - create_block(&mut conn, block_num_2); - - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num_2, - 0, - miden_objects::account::StorageSlotType::Value, - num_to_word(20), - ) - .unwrap(); - - // Reconstruct storage at block 1 - let storage_block_1 = - queries::select_account_storage_at_block(&mut conn, account_id, block_num_1).unwrap(); - match &storage_block_1.slots()[0] { - StorageSlot::Value(v) => assert_eq!(*v, num_to_word(10)), - StorageSlot::Map(_) => panic!("Expected Value slot"), - } - - // Reconstruct storage at block 2 - let storage_block_2 = - queries::select_account_storage_at_block(&mut conn, account_id, block_num_2).unwrap(); - match &storage_block_2.slots()[0] { - StorageSlot::Value(v) => assert_eq!(*v, num_to_word(20)), - StorageSlot::Map(_) => panic!("Expected Value slot"), - } - - // Reconstruct latest storage (should match block 2) - let storage_latest = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); - match &storage_latest.slots()[0] { - StorageSlot::Value(v) => assert_eq!(*v, num_to_word(20)), - StorageSlot::Map(_) => panic!("Expected Value slot"), - } -} - -#[test] -#[miden_node_test_macro::enable_logging] -fn test_storage_reconstruction_latest() { - let mut conn = create_db(); - - let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - let block_num = BlockNumber::from(1); - - // Create the block - create_block(&mut conn, block_num); - - // Insert storage headers: 2 Map slots and 1 Value slot - let map_commitment_1 = [Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]; - let map_commitment_2 = [Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]; - let value_slot = [Felt::new(100), Felt::new(200), Felt::new(300), Felt::new(400)]; - - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num, - 0, // slot 0: Map - StorageSlotType::Map, - map_commitment_1.into(), - ) - .unwrap(); - - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num, - 1, // slot 1: Map - StorageSlotType::Map, - map_commitment_2.into(), - ) - .unwrap(); - - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num, - 2, // slot 2: Value - StorageSlotType::Value, - value_slot.into(), - ) - .unwrap(); - - // Reconstruct storage from headers - let storage = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); - - // Verify we have 3 slots - assert_eq!(storage.slots().len(), 3); - - // Verify slot types - assert!(matches!(storage.slots()[0], miden_objects::account::StorageSlot::Map(_))); - assert!(matches!(storage.slots()[1], miden_objects::account::StorageSlot::Map(_))); - - if let miden_objects::account::StorageSlot::Value(value) = storage.slots()[2] { - assert_eq!(value, value_slot.into()); - } else { - panic!("Expected Value slot at index 2"); - } -} - -#[test] -#[miden_node_test_macro::enable_logging] -fn test_storage_reconstruction_historical() { - let mut conn = create_db(); - - let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - - // Block 1: Initial state with one value slot - let block_1 = BlockNumber::from(1); - create_block(&mut conn, block_1); - - let value_1 = [Felt::new(10), Felt::new(20), Felt::new(30), Felt::new(40)]; - queries::insert_account_storage_header( - &mut conn, - account_id, - block_1, - 0, - StorageSlotType::Value, - value_1.into(), - ) - .unwrap(); - - // Block 2: Update the value slot - let block_2 = BlockNumber::from(2); - create_block(&mut conn, block_2); - let value_2 = [Felt::new(50), Felt::new(60), Felt::new(70), Felt::new(80)]; - queries::insert_account_storage_header( - &mut conn, - account_id, - block_2, - 0, - StorageSlotType::Value, - value_2.into(), - ) - .unwrap(); - - // Reconstruct storage at block 1 - let storage_at_1 = - queries::select_account_storage_at_block(&mut conn, account_id, block_1).unwrap(); - assert_eq!(storage_at_1.slots().len(), 1); - if let miden_objects::account::StorageSlot::Value(value) = storage_at_1.slots()[0] { - assert_eq!(value, value_1.into()); - } else { - panic!("Expected Value slot"); - } - - // Reconstruct storage at block 2 - let storage_at_2 = - queries::select_account_storage_at_block(&mut conn, account_id, block_2).unwrap(); - assert_eq!(storage_at_2.slots().len(), 1); - if let miden_objects::account::StorageSlot::Value(value) = storage_at_2.slots()[0] { - assert_eq!(value, value_2.into()); - } else { - panic!("Expected Value slot"); - } - - // Latest should return block 2 value - let storage_latest = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); - assert_eq!(storage_latest.slots().len(), 1); - if let miden_objects::account::StorageSlot::Value(value) = storage_latest.slots()[0] { - assert_eq!(value, value_2.into()); - } else { - panic!("Expected Value slot"); - } -} - -#[test] -#[miden_node_test_macro::enable_logging] -fn test_storage_header_is_latest_flag() { - let mut conn = create_db(); - - let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - let slot_index = 0u8; - - let value_1 = [Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]; - let value_2 = [Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]; - let value_3 = [Felt::new(9), Felt::new(10), Felt::new(11), Felt::new(12)]; - - // Create the blocks - create_block(&mut conn, BlockNumber::from(1)); - create_block(&mut conn, BlockNumber::from(2)); - create_block(&mut conn, BlockNumber::from(3)); - - // Insert at block 1 - queries::insert_account_storage_header( - &mut conn, - account_id, - BlockNumber::from(1), - slot_index, - StorageSlotType::Value, - value_1.into(), - ) - .unwrap(); - - // Insert at block 2 - should mark block 1 as not latest - queries::insert_account_storage_header( - &mut conn, - account_id, - BlockNumber::from(2), - slot_index, - StorageSlotType::Value, - value_2.into(), - ) - .unwrap(); - - // Insert at block 3 - should mark block 2 as not latest - queries::insert_account_storage_header( - &mut conn, - account_id, - BlockNumber::from(3), - slot_index, - StorageSlotType::Value, - value_3.into(), - ) - .unwrap(); - - // Query latest - should return block 3 - let storage_latest = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); - assert_eq!(storage_latest.slots().len(), 1); - if let miden_objects::account::StorageSlot::Value(value) = storage_latest.slots()[0] { - assert_eq!(value, value_3.into()); - } else { - panic!("Expected Value slot with value_3"); - } - - // Verify historical queries still work - let storage_at_1 = - queries::select_account_storage_at_block(&mut conn, account_id, BlockNumber::from(1)) - .unwrap(); - if let miden_objects::account::StorageSlot::Value(value) = storage_at_1.slots()[0] { - assert_eq!(value, value_1.into()); - } else { - panic!("Expected Value slot with value_1"); - } -} - #[test] fn test_select_account_code_at_block() { let mut conn = create_db(); From a1173f77ded899fc5aa1c9955fd740eaa7849db5 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 11 Dec 2025 15:10:25 +0100 Subject: [PATCH 33/95] Revert "address review comments" This reverts commit 6336f41247a824cdfddd5eb2d488bb199716fc31. --- .../db/migrations/2025062000000_setup/up.sql | 25 +- .../store/src/db/models/queries/accounts.rs | 444 ++++++++++++---- .../src/db/models/queries/accounts/tests.rs | 478 ------------------ crates/store/src/db/schema.rs | 13 +- crates/store/src/db/tests.rs | 347 +++++++++++++ 5 files changed, 721 insertions(+), 586 deletions(-) delete mode 100644 crates/store/src/db/models/queries/accounts/tests.rs diff --git a/crates/store/src/db/migrations/2025062000000_setup/up.sql b/crates/store/src/db/migrations/2025062000000_setup/up.sql index e02f23e0c8..0512499262 100644 --- a/crates/store/src/db/migrations/2025062000000_setup/up.sql +++ b/crates/store/src/db/migrations/2025062000000_setup/up.sql @@ -19,15 +19,14 @@ CREATE TABLE accounts ( account_commitment BLOB NOT NULL, code_commitment BLOB, nonce INTEGER, - storage_header BLOB, -- Serialized AccountStorage from miden-objects is_latest BOOLEAN NOT NULL DEFAULT 0, -- Indicates if this is the latest state for this account_id PRIMARY KEY (account_id, block_num), CONSTRAINT all_null_or_none_null CHECK ( - (code_commitment IS NOT NULL AND nonce IS NOT NULL AND storage_header IS NOT NULL) + (code_commitment IS NOT NULL AND nonce IS NOT NULL) OR - (code_commitment IS NULL AND nonce IS NULL AND storage_header IS NULL) + (code_commitment IS NULL AND nonce IS NULL) ) ) WITHOUT ROWID; @@ -39,6 +38,26 @@ CREATE INDEX idx_accounts_block_num ON accounts(block_num); -- Index for joining with account_codes CREATE INDEX idx_accounts_code_commitment ON accounts(code_commitment) WHERE code_commitment IS NOT NULL; +-- Table to store storage slot headers (slot types and commitments) +CREATE TABLE account_storage_headers ( + account_id BLOB NOT NULL, + block_num INTEGER NOT NULL, + slot_index INTEGER NOT NULL, + slot_type INTEGER NOT NULL, -- 0=Map, 1=Value (as per StorageSlotType) + slot_commitment BLOB NOT NULL, + is_latest BOOLEAN NOT NULL DEFAULT 0, + + PRIMARY KEY (account_id, block_num, slot_index), + CONSTRAINT slot_index_is_u8 CHECK (slot_index BETWEEN 0 AND 0xFF), + CONSTRAINT slot_type_in_enum CHECK (slot_type BETWEEN 0 AND 1), + FOREIGN KEY (account_id, block_num) REFERENCES accounts(account_id, block_num) ON DELETE CASCADE +) WITHOUT ROWID; + +-- Index for joining with accounts table +CREATE INDEX idx_account_storage_headers_account_block ON account_storage_headers(account_id, block_num); +-- Index for querying latest state +CREATE INDEX idx_account_storage_headers_latest ON account_storage_headers(account_id, is_latest) WHERE is_latest = 1; + CREATE TABLE notes ( committed_at INTEGER NOT NULL, -- Block number when the note was committed batch_index INTEGER NOT NULL, -- Index of batch in block, starting from 0 diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 6ade45384b..313ca0c362 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -28,7 +28,9 @@ use miden_objects::account::{ AccountId, AccountStorage, NonFungibleDeltaAction, + StorageMap, StorageSlot, + StorageSlotType, }; use miden_objects::asset::{Asset, AssetVault, AssetVaultKey, FungibleAsset}; use miden_objects::block::{BlockAccountUpdate, BlockNumber}; @@ -483,52 +485,217 @@ pub(crate) fn select_account_storage_map_values( Ok(StorageMapValuesPage { last_block_included, values }) } -/// Returns account storage at a given block by deserializing the storage header blob. -/// Returns account storage at a given block by reading from `accounts.storage_header`. +/// Reconstruct a `StorageMap` from database entries using `SmtForest` +/// +/// This function builds an `SmtForest` from all key-value pairs at the specified block, +/// enabling efficient proof generation with structural sharing. The forest allows +/// maintaining multiple SMT versions in memory with shared nodes. +/// +/// # Arguments +/// +/// * `conn` - Database connection +/// * `account_id` - The account ID +/// * `block_num` - The block number +/// * `slot_index` - The storage slot index +/// +/// # Returns +/// +/// A reconstructed `StorageMap` backed by `SmtForest` with full proof capabilities. +pub(crate) fn reconstruct_storage_map_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, + slot_index: u8, +) -> Result { + use schema::account_storage_map_values as t; + + // Check if the requested block exists (returns error if not) + block_exists(conn, block_num)?; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + let slot_sql = slot_to_raw_sql(slot_index); + + // Query all entries for this slot at or before the given block + let raw: Vec<(Vec, Vec)> = SelectDsl::select(t::table, (t::key, t::value)) + .filter( + t::account_id + .eq(&account_id_bytes) + .and(t::slot.eq(slot_sql)) + .and(t::block_num.le(block_num_sql)), + ) + .load(conn)?; + + // Parse entries + let entries: Vec<(Word, Word)> = raw + .into_iter() + .map(|(k, v)| Ok((Word::read_from_bytes(&k)?, Word::read_from_bytes(&v)?))) + .collect::, DatabaseError>>()?; + + let entry_count = entries.len(); + + // StorageMap::with_entries internally uses an SMT which can be backed by SmtForest + // The SMT is built with structural sharing for memory efficiency + miden_objects::account::StorageMap::with_entries(entries).map_err(|e| { + DatabaseError::DataCorrupted(format!( + "Failed to create StorageMap from {entry_count} entries: {e}" + )) + }) +} + +/// Reconstruct `AccountStorage` from database tables for a specific account at a specific block +/// +/// This function queries the `account_storage_headers` table to get slot metadata and reconstructs +/// the `AccountStorage` without deserializing a blob. For Map slots, we only store the commitment +/// since the actual map data is in `account_storage_map_values`. +/// +/// # Returns +/// +/// The reconstructed `AccountStorage`, or an error if reconstruction fails. pub(crate) fn select_account_storage_at_block( conn: &mut SqliteConnection, account_id: AccountId, block_num: BlockNumber, ) -> Result { + use schema::account_storage_headers as t; + + // Check if the requested block exists (returns error if not) block_exists(conn, block_num)?; - let storage_header_blob: Option> = - SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) - .filter( - schema::accounts::account_id - .eq(account_id.to_bytes()) - .and(schema::accounts::block_num.le(block_num.to_raw_sql())), - ) - .order(schema::accounts::block_num.desc()) - .limit(1) - .first(conn) - .optional()? - .flatten(); - - storage_header_blob - .map(|blob| AccountStorage::read_from_bytes(&blob).map_err(Into::into)) - .unwrap_or_else(|| Ok(AccountStorage::new(Vec::new())?)) + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + + // Query storage headers for this account at this block + let headers: Vec = + SelectDsl::select(t::table, AccountStorageHeaderRaw::as_select()) + .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.eq(block_num_sql))) + .order(t::slot_index.asc()) + .load(conn)?; + + if headers.is_empty() { + // No storage headers means empty storage + return Ok(AccountStorage::new(Vec::new())?); + } + + // Build slots from headers + let mut slots = Vec::with_capacity(headers.len()); + + for header in headers { + let slot_type = StorageSlotType::from_raw_sql(header.slot_type)?; + + let commitment = Word::read_from_bytes(&header.slot_commitment)?; + + let slot = match slot_type { + StorageSlotType::Map => { + // For Map slots, we create an empty map + // The actual map data is queried separately when needed from + // account_storage_map_values + + // Create an empty storage map + let storage_map = StorageMap::new(); + StorageSlot::Map(storage_map) + }, + StorageSlotType::Value => { + // For Value slots, the commitment IS the value + StorageSlot::Value(commitment) + }, + }; + + slots.push(slot); + } + + Ok(AccountStorage::new(slots)?) } -/// Select latest account storage by querying `accounts` where `is_latest=true`. +/// Select account storage headers at a specific block (lightweight query). +/// +/// Returns tuples of `(slot_index, slot_type, commitment)` without reconstructing full slots. +#[allow(dead_code)] // Helper for future SmtForest integration +pub(crate) fn select_account_storage_headers_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { + use schema::account_storage_headers as t; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + + let headers: Vec = + SelectDsl::select(t::table, AccountStorageHeaderRaw::as_select()) + .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.le(block_num_sql))) + .order(t::slot_index.asc()) + .load(conn)?; + + headers + .into_iter() + .map(|h| { + let slot_index = raw_sql_to_slot(h.slot_index); + let slot_type = StorageSlotType::from_raw_sql(h.slot_type)?; + let commitment = Word::read_from_bytes(&h.slot_commitment)?; + Ok((slot_index, slot_type, commitment)) + }) + .collect() +} + +/// Reconstruct `AccountStorage` from the latest state in the database +/// +/// This queries only the latest storage headers (where `is_latest=true`) for faster reconstruction +/// Select the latest storage headers for an account +/// +/// This function queries the `account_storage_headers` table for the latest state of an account's +/// storage slots, using the `is_latest=true` flag for efficiency. +/// +/// # Returns +/// +/// The reconstructed `AccountStorage` from the latest storage headers. pub(crate) fn select_latest_account_storage( conn: &mut SqliteConnection, account_id: AccountId, ) -> Result { - let storage_header_blob: Option> = - SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) - .filter( - schema::accounts::account_id - .eq(account_id.to_bytes()) - .and(schema::accounts::is_latest.eq(true)), - ) - .first(conn) - .optional()? - .flatten(); + use schema::account_storage_headers as t; + + let account_id_bytes = account_id.to_bytes(); - storage_header_blob - .map(|blob| AccountStorage::read_from_bytes(&blob).map_err(Into::into)) - .unwrap_or_else(|| Ok(AccountStorage::new(Vec::new())?)) + // Query latest storage headers for this account + let headers: Vec = + SelectDsl::select(t::table, AccountStorageHeaderRaw::as_select()) + .filter(t::account_id.eq(&account_id_bytes).and(t::is_latest.eq(true))) + .order(t::slot_index.asc()) + .load(conn)?; + + if headers.is_empty() { + // No storage headers means empty storage + return Ok(AccountStorage::new(Vec::new())?); + } + + // Build slots from headers + let mut slots = Vec::with_capacity(headers.len()); + + for header in headers { + let slot_type = StorageSlotType::from_raw_sql(header.slot_type)?; + let slot_index = raw_sql_to_slot(header.slot_index); + let block_num = BlockNumber::from_raw_sql(header.block_num)?; + let commitment = Word::read_from_bytes(&header.slot_commitment)?; + + let slot = match slot_type { + StorageSlotType::Map => { + // For Map slots, reconstruct the full SMT from database entries + // This allows serving proofs for any key in the map + let storage_map = + reconstruct_storage_map_at_block(conn, account_id, block_num, slot_index)?; + StorageSlot::Map(storage_map) + }, + StorageSlotType::Value => { + // For Value slots, the commitment IS the value + StorageSlot::Value(commitment) + }, + }; + + slots.push(slot); + } + + Ok(AccountStorage::new(slots)?) } #[derive(Queryable, Selectable)] @@ -552,6 +719,19 @@ impl TryFrom for AccountVaultValue { } } +#[derive(Debug, Clone, Queryable, Selectable)] +#[diesel(table_name = schema::account_storage_headers)] +#[diesel(check_for_backend(diesel::sqlite::Sqlite))] +#[allow(dead_code)] // Fields used by Diesel, not directly in Rust code +pub struct AccountStorageHeaderRaw { + pub account_id: Vec, + pub block_num: i64, + pub slot_index: i32, + pub slot_type: i32, + pub slot_commitment: Vec, + pub is_latest: bool, +} + #[derive(Debug, Clone, PartialEq, Eq, Selectable, Queryable, QueryableByName)] #[diesel(table_name = schema::accounts)] #[diesel(check_for_backend(Sqlite))] @@ -616,6 +796,59 @@ pub(crate) fn insert_account_vault_asset( }) } +/// Insert an account storage header into the DB using the given [`SqliteConnection`]. +/// +/// Sets `is_latest=true` for the new row and updates any existing +/// row with the same (`account_id`, `slot_index`) tuple to `is_latest=false`. +/// +/// # Returns +/// +/// The number of affected rows. +#[cfg(test)] +pub(crate) fn insert_account_storage_header( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, + slot_index: u8, + slot_type: StorageSlotType, + slot_commitment: Word, +) -> Result { + use schema::account_storage_headers as t; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + let slot_index_sql = slot_to_raw_sql(slot_index); + let slot_type_sql = slot_type.to_raw_sql(); + let slot_commitment_bytes = slot_commitment.to_bytes(); + + diesel::Connection::transaction(conn, |conn| { + // Update existing headers for this slot to set is_latest=false + let update_count = diesel::update(t::table) + .filter( + t::account_id + .eq(&account_id_bytes) + .and(t::slot_index.eq(slot_index_sql)) + .and(t::is_latest.eq(true)), + ) + .set(t::is_latest.eq(false)) + .execute(conn)?; + + // Insert the new latest row + let insert_count = diesel::insert_into(t::table) + .values(( + t::account_id.eq(&account_id_bytes), + t::block_num.eq(block_num_sql), + t::slot_index.eq(slot_index_sql), + t::slot_type.eq(slot_type_sql), + t::slot_commitment.eq(&slot_commitment_bytes), + t::is_latest.eq(true), + )) + .execute(conn)?; + + Ok(update_count + insert_count) + }) +} + /// Insert an account storage map value into the DB using the given [`SqliteConnection`]. /// /// Sets `is_latest=true` for the new row and updates any existing @@ -857,7 +1090,6 @@ pub(crate) fn upsert_accounts( code_commitment: full_account .as_ref() .map(|account| account.code().commitment().to_bytes()), - storage_header: full_account.as_ref().map(|account| account.storage().to_bytes()), is_latest: true, }; @@ -865,10 +1097,12 @@ pub(crate) fn upsert_accounts( .values(&account_value) .execute(conn)?; + // insert pending storage map entries for (acc_id, slot, key, value) in pending_storage_inserts { insert_account_storage_map_value(conn, acc_id, block_num, slot, key, value)?; } + // insert pending vault-asset entries for (acc_id, vault_key, update) in pending_asset_inserts { insert_account_vault_asset(conn, acc_id, block_num, vault_key, update)?; } @@ -905,6 +1139,17 @@ pub(crate) struct AccountCodeRowInsert { pub(crate) code: Vec, } +#[derive(Insertable, AsChangeset, Debug, Clone)] +#[diesel(table_name = schema::account_storage_headers)] +pub(crate) struct AccountStorageHeaderInsert { + pub(crate) account_id: Vec, + pub(crate) block_num: i64, + pub(crate) slot_index: i32, + pub(crate) slot_type: i32, + pub(crate) slot_commitment: Vec, + pub(crate) is_latest: bool, +} + #[derive(Insertable, AsChangeset, Debug, Clone)] #[diesel(table_name = schema::accounts)] pub(crate) struct AccountRowInsert { @@ -914,7 +1159,6 @@ pub(crate) struct AccountRowInsert { pub(crate) account_commitment: Vec, pub(crate) code_commitment: Option>, pub(crate) nonce: Option, - pub(crate) storage_header: Option>, pub(crate) is_latest: bool, } @@ -962,7 +1206,7 @@ pub(crate) struct AccountStorageMapRowInsert { pub(crate) is_latest: bool, } -/// Query vault assets at a specific block by finding the most recent update for each `vault_key`. +/// Queries vault assets (key, value) pairs at a specific block pub(crate) fn select_account_vault_at_block( conn: &mut SqliteConnection, account_id: AccountId, @@ -974,54 +1218,48 @@ pub(crate) fn select_account_vault_at_block( block_exists(conn, block_num)?; let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - - // Since Diesel doesn't support composite keys in subqueries easily, we use a two-step approach: - // Step 1: Get max block_num for each vault_key - let max_blocks: Vec<(Vec, i64)> = QueryDsl::select( + let block_num_sql = i64::from(block_num.as_u32()); + let raw: Vec<(Vec, Option>)> = SelectDsl::select( t::table .filter(t::account_id.eq(&account_id_bytes)) .filter(t::block_num.le(block_num_sql)) - .group_by(t::vault_key), - (t::vault_key, diesel::dsl::max(t::block_num)), + .order(t::block_num.desc()) + .limit(1), + (t::vault_key, t::asset), ) - .load::<(Vec, Option)>(conn)? - .into_iter() - .filter_map(|(key, maybe_block)| maybe_block.map(|block| (key, block))) - .collect(); + .load(conn)?; - if max_blocks.is_empty() { - return Ok(Vec::new()); - } + let entries = raw + .into_iter() + .filter_map(|(key_bytes, maybe_asset_bytes)| { + let key = Word::read_from_bytes(&key_bytes).ok()?; + let asset_bytes = maybe_asset_bytes?; + let value = Word::read_from_bytes(&asset_bytes).ok()?; + Some((key, value)) + }) + .collect(); - // Step 2: Fetch the full rows matching (vault_key, block_num) pairs - let mut entries = Vec::new(); - for (vault_key_bytes, max_block) in max_blocks { - let result: Option<(Vec, Option>)> = QueryDsl::select( - t::table.filter( - t::account_id - .eq(&account_id_bytes) - .and(t::vault_key.eq(&vault_key_bytes)) - .and(t::block_num.eq(max_block)), - ), - (t::vault_key, t::asset), - ) - .first(conn) - .optional()?; - - if let Some((key_bytes, Some(asset_bytes))) = result { - if let (Ok(key), Ok(value)) = - (Word::read_from_bytes(&key_bytes), Word::read_from_bytes(&asset_bytes)) - { - entries.push((key, value)); - } - } - } + Ok(entries) +} + +/// Computes the storage commitment from a list of slot commitments. +/// +/// This replicates the logic from `AccountStorage::commitment()` which hashes all slot +/// commitments together. +/// +/// # Arguments +/// +/// * `slot_commitments` - Vector of slot commitment words +/// +/// # Returns +/// +/// The storage commitment as a `Word` +fn compute_storage_commitment(slot_commitments: &[Word]) -> Word { + use miden_objects::crypto::hash::rpo::Rpo256; - // Sort by vault_key for consistent ordering - entries.sort_by_key(|(key, _)| *key); + let elements: Vec = slot_commitments.iter().flat_map(|w| w.iter()).copied().collect(); - Ok(entries) + Rpo256::hash_elements(&elements) } /// Helper function to check if a block exists in the `block_headers` table. @@ -1101,18 +1339,12 @@ pub(crate) fn select_account_code_at_block( Ok(result) } -#[derive(Debug, Clone, Queryable)] -struct AccountHeaderDataRaw { - code_commitment: Option>, - nonce: Option, - storage_header: Option>, -} - /// Queries the account header for a specific account at a specific block number. /// /// This reconstructs the `AccountHeader` by joining multiple tables: -/// - `accounts` table for `account_id`, `nonce`, `code_commitment`, `storage_header` +/// - `accounts` table for `account_id`, `nonce`, `code_commitment` /// - `account_vault_headers` table for `vault_root` +/// - `account_storage_headers` table for storage slot commitments (to compute `storage_commitment`) /// /// Returns `None` if the account doesn't exist at that block. /// @@ -1132,33 +1364,27 @@ pub(crate) fn select_account_header_at_block( account_id: AccountId, block_num: BlockNumber, ) -> Result, DatabaseError> { - use schema::{account_vault_headers, accounts}; + use schema::{account_storage_headers, account_vault_headers, accounts}; + // Check if the requested block exists (returns error if not) block_exists(conn, block_num)?; let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); - - let account_data: Option = SelectDsl::select( + let account_data: Option<(Option>, Option)> = SelectDsl::select( accounts::table .filter(accounts::account_id.eq(&account_id_bytes)) .filter(accounts::block_num.le(block_num_sql)) .order(accounts::block_num.desc()) .limit(1), - (accounts::code_commitment, accounts::nonce, accounts::storage_header), + (accounts::code_commitment, accounts::nonce), ) .first(conn) .optional()?; - let Some(AccountHeaderDataRaw { - code_commitment: code_commitment_bytes, - nonce: nonce_raw, - storage_header: storage_header_blob, - }) = account_data - else { + let Some((code_commitment_bytes, nonce_raw)) = account_data else { return Ok(None); }; - let vault_root_bytes: Option> = SelectDsl::select( account_vault_headers::table .filter(account_vault_headers::account_id.eq(&account_id_bytes)) @@ -1170,13 +1396,26 @@ pub(crate) fn select_account_header_at_block( .first(conn) .optional()?; - let storage_commitment = match storage_header_blob { - Some(blob) => { - let storage = AccountStorage::read_from_bytes(&blob)?; - storage.commitment() - }, - None => Word::default(), - }; + let storage_slots: Vec<(i32, i32, Vec)> = SelectDsl::select( + account_storage_headers::table + .filter(account_storage_headers::account_id.eq(&account_id_bytes)) + .filter(account_storage_headers::block_num.le(block_num_sql)) + .order(account_storage_headers::block_num.desc()) + .limit(1), + ( + account_storage_headers::slot_index, + account_storage_headers::slot_type, + account_storage_headers::slot_commitment, + ), + ) + .load(conn)?; + + let slot_commitments: Vec = storage_slots + .into_iter() + .map(|(_slot_index, _slot_type, commitment_bytes)| Word::read_from_bytes(&commitment_bytes)) + .collect::, _>>()?; + + let storage_commitment = compute_storage_commitment(&slot_commitments); let code_commitment = code_commitment_bytes .map(|bytes| Word::read_from_bytes(&bytes)) @@ -1198,6 +1437,3 @@ pub(crate) fn select_account_header_at_block( code_commitment, ))) } - -#[cfg(test)] -mod tests; diff --git a/crates/store/src/db/models/queries/accounts/tests.rs b/crates/store/src/db/models/queries/accounts/tests.rs deleted file mode 100644 index b68df7367e..0000000000 --- a/crates/store/src/db/models/queries/accounts/tests.rs +++ /dev/null @@ -1,478 +0,0 @@ -use assert_matches::assert_matches; -use diesel::{Connection, RunQueryDsl}; -use diesel_migrations::MigrationHarness; -use miden_lib::account::auth::AuthRpoFalcon512; -use miden_lib::transaction::TransactionKernel; -use miden_node_utils::fee::test_fee_params; -use miden_objects::account::auth::PublicKeyCommitment; -use miden_objects::account::{ - AccountBuilder, - AccountComponent, - AccountIdVersion, - AccountStorageMode, - AccountType, - StorageSlot, -}; -use miden_objects::{EMPTY_WORD, Word}; - -use super::*; -use crate::db::migrations::MIGRATIONS; - -fn setup_test_db() -> SqliteConnection { - let mut conn = - SqliteConnection::establish(":memory:").expect("Failed to create in-memory database"); - - conn.run_pending_migrations(MIGRATIONS).expect("Failed to run migrations"); - - conn -} - -fn create_test_account_with_storage() -> (Account, AccountId) { - // Create a simple public account with one value storage slot - let account_id = AccountId::dummy( - [1u8; 15], - AccountIdVersion::Version0, - AccountType::RegularAccountImmutableCode, - AccountStorageMode::Public, - ); - - let storage_value = Word::from([Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]); - let component_storage = vec![StorageSlot::Value(storage_value)]; - - let component = AccountComponent::compile( - "export.foo push.1 end", - TransactionKernel::assembler(), - component_storage, - ) - .unwrap() - .with_supported_type(AccountType::RegularAccountImmutableCode); - - let account = AccountBuilder::new([1u8; 32]) - .account_type(AccountType::RegularAccountImmutableCode) - .storage_mode(AccountStorageMode::Public) - .with_component(component) - .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) - .build_existing() - .unwrap(); - - (account, account_id) -} - -fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { - use miden_objects::block::BlockHeader; - - use crate::db::schema::block_headers; - - let block_header = BlockHeader::new( - 1_u8.into(), - Word::default(), - block_num, - Word::default(), - Word::default(), - Word::default(), - Word::default(), - Word::default(), - Word::default(), - Word::default(), - test_fee_params(), - 0_u8.into(), - ); - - diesel::insert_into(block_headers::table) - .values(( - block_headers::block_num.eq(i64::from(block_num.as_u32())), - block_headers::block_header.eq(block_header.to_bytes()), - )) - .execute(conn) - .expect("Failed to insert block header"); -} - -#[test] -fn test_upsert_accounts_inserts_storage_header() { - let mut conn = setup_test_db(); - let (account, account_id) = create_test_account_with_storage(); - - // Block 1 - let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); - - let storage_commitment_original = account.storage().commitment(); - let storage_slots_len = account.storage().slots().len(); - let account_commitment = account.commitment(); - - // Create full state delta from the account - let delta = AccountDelta::try_from(account).unwrap(); - assert!(delta.is_full_state(), "Delta should be full state"); - - let account_update = - BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Delta(delta)); - - // Upsert account - let result = upsert_accounts(&mut conn, &[account_update], block_num); - assert!(result.is_ok(), "upsert_accounts failed: {:?}", result.err()); - assert_eq!(result.unwrap(), 1, "Expected 1 account to be inserted"); - - // Query storage header back - let queried_storage = select_latest_account_storage(&mut conn, account_id) - .expect("Failed to query storage header"); - - // Verify storage commitment matches - assert_eq!( - queried_storage.commitment(), - storage_commitment_original, - "Storage commitment mismatch" - ); - - // Verify number of slots matches - assert_eq!(queried_storage.slots().len(), storage_slots_len, "Storage slots count mismatch"); - - // Verify exactly 1 latest account with storage exists - let header_count: i64 = schema::accounts::table - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::is_latest.eq(true)) - .filter(schema::accounts::storage_header.is_not_null()) - .count() - .get_result(&mut conn) - .expect("Failed to count accounts with storage"); - - assert_eq!(header_count, 1, "Expected exactly 1 latest account with storage"); -} - -#[test] -fn test_upsert_accounts_updates_is_latest_flag() { - let mut conn = setup_test_db(); - let (account, account_id) = create_test_account_with_storage(); - - // Block 1 and 2 - let block_num_1 = BlockNumber::from_epoch(0); - let block_num_2 = BlockNumber::from_epoch(1); - - insert_block_header(&mut conn, block_num_1); - insert_block_header(&mut conn, block_num_2); - - // Save storage commitment before moving account - let storage_commitment_1 = account.storage().commitment(); - let account_commitment_1 = account.commitment(); - - // First update with original account - full state delta - let delta_1 = AccountDelta::try_from(account).unwrap(); - - let account_update_1 = BlockAccountUpdate::new( - account_id, - account_commitment_1, - AccountUpdateDetails::Delta(delta_1), - ); - - upsert_accounts(&mut conn, &[account_update_1], block_num_1).expect("First upsert failed"); - - // Create modified account with different storage value - let storage_value_modified = - Word::from([Felt::new(10), Felt::new(20), Felt::new(30), Felt::new(40)]); - let component_storage_modified = vec![StorageSlot::Value(storage_value_modified)]; - - let component_2 = AccountComponent::compile( - "export.foo push.1 end", - TransactionKernel::assembler(), - component_storage_modified, - ) - .unwrap() - .with_supported_type(AccountType::RegularAccountImmutableCode); - - let account_2 = AccountBuilder::new([1u8; 32]) - .account_type(AccountType::RegularAccountImmutableCode) - .storage_mode(AccountStorageMode::Public) - .with_component(component_2) - .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) - .build_existing() - .unwrap(); - - let storage_commitment_2 = account_2.storage().commitment(); - let account_commitment_2 = account_2.commitment(); - - // Second update with modified account - full state delta - let delta_2 = AccountDelta::try_from(account_2).unwrap(); - - let account_update_2 = BlockAccountUpdate::new( - account_id, - account_commitment_2, - AccountUpdateDetails::Delta(delta_2), - ); - - upsert_accounts(&mut conn, &[account_update_2], block_num_2).expect("Second upsert failed"); - - // Verify 2 total account rows exist (both historical records) - let total_accounts: i64 = schema::accounts::table - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .count() - .get_result(&mut conn) - .expect("Failed to count total accounts"); - - assert_eq!(total_accounts, 2, "Expected 2 total account records"); - - // Verify only 1 is marked as latest - let latest_accounts: i64 = schema::accounts::table - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::is_latest.eq(true)) - .count() - .get_result(&mut conn) - .expect("Failed to count latest accounts"); - - assert_eq!(latest_accounts, 1, "Expected exactly 1 latest account"); - - // Verify latest storage matches second update - let latest_storage = select_latest_account_storage(&mut conn, account_id) - .expect("Failed to query latest storage"); - - assert_eq!( - latest_storage.commitment(), - storage_commitment_2, - "Latest storage should match second update" - ); - - // Verify historical query returns first update - let storage_at_block_1 = select_account_storage_at_block(&mut conn, account_id, block_num_1) - .expect("Failed to query storage at block 1"); - - assert_eq!( - storage_at_block_1.commitment(), - storage_commitment_1, - "Storage at block 1 should match first update" - ); -} - -#[test] -fn test_upsert_accounts_with_incremental_delta() { - use std::collections::BTreeMap; - - use miden_objects::account::delta::{AccountStorageDelta, AccountVaultDelta}; - - let mut conn = setup_test_db(); - let (account, account_id) = create_test_account_with_storage(); - - let block_num_1 = BlockNumber::from_epoch(0); - let block_num_2 = BlockNumber::from_epoch(1); - - insert_block_header(&mut conn, block_num_1); - insert_block_header(&mut conn, block_num_2); - - // First update with full state - let storage_commitment_1 = account.storage().commitment(); - let account_commitment_1 = account.commitment(); - let nonce_1 = account.nonce(); - let delta_1 = AccountDelta::try_from(account).unwrap(); - - let account_update_1 = BlockAccountUpdate::new( - account_id, - account_commitment_1, - AccountUpdateDetails::Delta(delta_1), - ); - - upsert_accounts(&mut conn, &[account_update_1], block_num_1).expect("First upsert failed"); - - // Create incremental delta (only modify storage value slot 1) - let new_storage_value = - Word::from([Felt::new(100), Felt::new(200), Felt::new(300), Felt::new(400)]); - - let mut storage_delta_values = BTreeMap::new(); - storage_delta_values.insert(1u8, new_storage_value); // Update slot 1 (component storage) - - let storage_delta = AccountStorageDelta::from_parts(storage_delta_values, BTreeMap::new()) - .expect("Failed to create storage delta"); - let incremental_delta = - AccountDelta::new(account_id, storage_delta, AccountVaultDelta::default(), nonce_1) - .expect("Failed to create incremental delta"); - - // Reconstruct expected account after delta - let account_after = reconstruct_full_account_from_db(&mut conn, account_id) - .expect("Failed to reconstruct account"); - let mut expected_account = account_after.clone(); - expected_account - .apply_delta(&incremental_delta) - .expect("Failed to apply delta to expected account"); - - let storage_commitment_2 = expected_account.storage().commitment(); - let account_commitment_2 = expected_account.commitment(); - - let account_update_2 = BlockAccountUpdate::new( - account_id, - account_commitment_2, - AccountUpdateDetails::Delta(incremental_delta), - ); - - upsert_accounts(&mut conn, &[account_update_2], block_num_2) - .expect("Second upsert with incremental delta failed"); - - // Verify latest storage matches expected state - let latest_storage = select_latest_account_storage(&mut conn, account_id) - .expect("Failed to query latest storage"); - - assert_eq!( - latest_storage.commitment(), - storage_commitment_2, - "Storage commitment should match after incremental delta" - ); - - // Verify historical storage is preserved - let storage_at_block_1 = select_account_storage_at_block(&mut conn, account_id, block_num_1) - .expect("Failed to query storage at block 1"); - - assert_eq!( - storage_at_block_1.commitment(), - storage_commitment_1, - "Historical storage should be unchanged" - ); -} - -#[test] -fn test_upsert_accounts_with_multiple_storage_slots() { - let mut conn = setup_test_db(); - - // Create account with 3 storage slots - let account_id = AccountId::dummy( - [2u8; 15], - AccountIdVersion::Version0, - AccountType::RegularAccountImmutableCode, - AccountStorageMode::Public, - ); - - let slot_value_1 = Word::from([Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]); - let slot_value_2 = Word::from([Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]); - let slot_value_3 = Word::from([Felt::new(9), Felt::new(10), Felt::new(11), Felt::new(12)]); - - let component_storage = vec![ - StorageSlot::Value(slot_value_1), - StorageSlot::Value(slot_value_2), - StorageSlot::Value(slot_value_3), - ]; - - let component = AccountComponent::compile( - "export.foo push.1 end", - TransactionKernel::assembler(), - component_storage, - ) - .unwrap() - .with_supported_type(AccountType::RegularAccountImmutableCode); - - let account = AccountBuilder::new([2u8; 32]) - .account_type(AccountType::RegularAccountImmutableCode) - .storage_mode(AccountStorageMode::Public) - .with_component(component) - .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) - .build_existing() - .unwrap(); - - let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); - - let storage_commitment = account.storage().commitment(); - let account_commitment = account.commitment(); - let delta = AccountDelta::try_from(account).unwrap(); - - let account_update = - BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Delta(delta)); - - upsert_accounts(&mut conn, &[account_update], block_num) - .expect("Upsert with multiple storage slots failed"); - - // Query back and verify - let queried_storage = - select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); - - assert_eq!(queried_storage.commitment(), storage_commitment, "Storage commitment mismatch"); - - // Note: Auth component adds 1 storage slot, so 3 component slots + 1 auth = 4 total - assert_eq!( - queried_storage.slots().len(), - 4, - "Expected 4 storage slots (3 component + 1 auth)" - ); - - // Verify individual slot values (skipping auth slot at index 0) - assert_matches!( - queried_storage.slots().get(1).expect("Slot 1 should exist"), - &StorageSlot::Value(v) if v == slot_value_1, - "Slot 1 value mismatch" - ); - assert_matches!( - queried_storage.slots().get(2).expect("Slot 2 should exist"), - &StorageSlot::Value(v) if v == slot_value_2, - "Slot 2 value mismatch" - ); - assert_matches!( - queried_storage.slots().get(3).expect("Slot 3 should exist"), - &StorageSlot::Value(v) if v == slot_value_3, - "Slot 3 value mismatch" - ); -} - -#[test] -fn test_upsert_accounts_with_empty_storage() { - let mut conn = setup_test_db(); - - // Create account with no storage slots - let account_id = AccountId::dummy( - [3u8; 15], - AccountIdVersion::Version0, - AccountType::RegularAccountImmutableCode, - AccountStorageMode::Public, - ); - - let component = AccountComponent::compile( - "export.foo push.1 end", - TransactionKernel::assembler(), - vec![], // Empty storage - ) - .unwrap() - .with_supported_type(AccountType::RegularAccountImmutableCode); - - let account = AccountBuilder::new([3u8; 32]) - .account_type(AccountType::RegularAccountImmutableCode) - .storage_mode(AccountStorageMode::Public) - .with_component(component) - .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) - .build_existing() - .unwrap(); - - let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); - - let storage_commitment = account.storage().commitment(); - let account_commitment = account.commitment(); - let delta = AccountDelta::try_from(account).unwrap(); - - let account_update = - BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Delta(delta)); - - upsert_accounts(&mut conn, &[account_update], block_num) - .expect("Upsert with empty storage failed"); - - // Query back and verify - let queried_storage = - select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); - - assert_eq!( - queried_storage.commitment(), - storage_commitment, - "Storage commitment mismatch for empty storage" - ); - - // Note: Auth component adds 1 storage slot, so even "empty" accounts have 1 slot - assert_eq!(queried_storage.slots().len(), 1, "Expected 1 storage slot (auth component)"); - - // Verify the storage header blob exists in database - let storage_header_exists: Option = SelectDsl::select( - schema::accounts::table - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::is_latest.eq(true)), - schema::accounts::storage_header.is_not_null(), - ) - .first(&mut conn) - .optional() - .expect("Failed to check storage header existence"); - - assert_eq!( - storage_header_exists, - Some(true), - "Storage header blob should exist even for empty storage" - ); -} diff --git a/crates/store/src/db/schema.rs b/crates/store/src/db/schema.rs index 18d557bdde..4929d3e106 100644 --- a/crates/store/src/db/schema.rs +++ b/crates/store/src/db/schema.rs @@ -1,5 +1,16 @@ // @generated automatically by Diesel CLI. +diesel::table! { + account_storage_headers (account_id, block_num, slot_index) { + account_id -> Binary, + block_num -> BigInt, + slot_index -> Integer, + slot_type -> Integer, + slot_commitment -> Binary, + is_latest -> Bool, + } +} + diesel::table! { account_storage_map_values (account_id, block_num, slot, key) { account_id -> Binary, @@ -37,7 +48,6 @@ diesel::table! { account_commitment -> Binary, code_commitment -> Nullable, nonce -> Nullable, - storage_header -> Nullable, block_num -> BigInt, is_latest -> Bool, } @@ -120,6 +130,7 @@ diesel::joinable!(transactions -> block_headers (block_num)); diesel::allow_tables_to_appear_in_same_query!( account_codes, + account_storage_headers, account_storage_map_values, accounts, account_vault_assets, diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index e4910b1a43..12263c0f5b 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -20,11 +20,14 @@ use miden_objects::account::{ AccountDelta, AccountId, AccountIdVersion, + AccountStorage, AccountStorageDelta, AccountStorageMode, AccountType, AccountVaultDelta, + StorageMap, StorageSlot, + StorageSlotType, }; use miden_objects::asset::{Asset, AssetVaultKey, FungibleAsset}; use miden_objects::block::{ @@ -1538,6 +1541,350 @@ fn mock_account_code_and_storage( // STORAGE RECONSTRUCTION TESTS // ================================================================================================ +#[test] +#[miden_node_test_macro::enable_logging] +fn test_storage_reconstruction_latest_state() { + let mut conn = create_db(); + + // Create an account with storage slots + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + let block_num = BlockNumber::from(1); + + // Create the block before inserting storage data + create_block(&mut conn, block_num); + + // Create test storage with Value and Map slots + let value_slot = StorageSlot::Value(num_to_word(42)); + let mut storage_map = StorageMap::new(); + let _ = storage_map.insert(num_to_word(1), num_to_word(100)); + let _ = storage_map.insert(num_to_word(2), num_to_word(200)); + let map_slot = StorageSlot::Map(storage_map.clone()); + + let _storage = AccountStorage::new(vec![value_slot, map_slot]).unwrap(); + + // Insert storage headers for both slots + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num, + 0, // slot_index + miden_objects::account::StorageSlotType::Value, + num_to_word(42), + ) + .unwrap(); + + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num, + 1, // slot_index + miden_objects::account::StorageSlotType::Map, + storage_map.root(), + ) + .unwrap(); + + // Insert map values + queries::insert_account_storage_map_value( + &mut conn, + account_id, + block_num, + 1, // slot + num_to_word(1), // key + num_to_word(100), // value + ) + .unwrap(); + + queries::insert_account_storage_map_value( + &mut conn, + account_id, + block_num, + 1, // slot + num_to_word(2), // key + num_to_word(200), // value + ) + .unwrap(); + + // Reconstruct storage from latest state + let reconstructed_storage = + queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + + // Verify reconstructed storage + assert_eq!(reconstructed_storage.slots().len(), 2); + + // Check Value slot + match &reconstructed_storage.slots()[0] { + StorageSlot::Value(v) => assert_eq!(*v, num_to_word(42)), + StorageSlot::Map(_) => panic!("Expected Value slot"), + } + + // Check Map slot (commitment should match) + match &reconstructed_storage.slots()[1] { + StorageSlot::Map(_) => { + // The map should be reconstructed (empty but with correct slot type) + // Actual values would need to be queried separately from account_storage_map_values + }, + StorageSlot::Value(_) => panic!("Expected Map slot"), + } +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn test_storage_reconstruction_historical_state() { + let mut conn = create_db(); + + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + + // Block 1: Initial storage + let block_num_1 = BlockNumber::from(1); + create_block(&mut conn, block_num_1); + + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num_1, + 0, + miden_objects::account::StorageSlotType::Value, + num_to_word(10), + ) + .unwrap(); + + // Block 2: Updated storage + let block_num_2 = BlockNumber::from(2); + create_block(&mut conn, block_num_2); + + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num_2, + 0, + miden_objects::account::StorageSlotType::Value, + num_to_word(20), + ) + .unwrap(); + + // Reconstruct storage at block 1 + let storage_block_1 = + queries::select_account_storage_at_block(&mut conn, account_id, block_num_1).unwrap(); + match &storage_block_1.slots()[0] { + StorageSlot::Value(v) => assert_eq!(*v, num_to_word(10)), + StorageSlot::Map(_) => panic!("Expected Value slot"), + } + + // Reconstruct storage at block 2 + let storage_block_2 = + queries::select_account_storage_at_block(&mut conn, account_id, block_num_2).unwrap(); + match &storage_block_2.slots()[0] { + StorageSlot::Value(v) => assert_eq!(*v, num_to_word(20)), + StorageSlot::Map(_) => panic!("Expected Value slot"), + } + + // Reconstruct latest storage (should match block 2) + let storage_latest = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + match &storage_latest.slots()[0] { + StorageSlot::Value(v) => assert_eq!(*v, num_to_word(20)), + StorageSlot::Map(_) => panic!("Expected Value slot"), + } +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn test_storage_reconstruction_latest() { + let mut conn = create_db(); + + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + let block_num = BlockNumber::from(1); + + // Create the block + create_block(&mut conn, block_num); + + // Insert storage headers: 2 Map slots and 1 Value slot + let map_commitment_1 = [Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]; + let map_commitment_2 = [Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]; + let value_slot = [Felt::new(100), Felt::new(200), Felt::new(300), Felt::new(400)]; + + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num, + 0, // slot 0: Map + StorageSlotType::Map, + map_commitment_1.into(), + ) + .unwrap(); + + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num, + 1, // slot 1: Map + StorageSlotType::Map, + map_commitment_2.into(), + ) + .unwrap(); + + queries::insert_account_storage_header( + &mut conn, + account_id, + block_num, + 2, // slot 2: Value + StorageSlotType::Value, + value_slot.into(), + ) + .unwrap(); + + // Reconstruct storage from headers + let storage = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + + // Verify we have 3 slots + assert_eq!(storage.slots().len(), 3); + + // Verify slot types + assert!(matches!(storage.slots()[0], miden_objects::account::StorageSlot::Map(_))); + assert!(matches!(storage.slots()[1], miden_objects::account::StorageSlot::Map(_))); + + if let miden_objects::account::StorageSlot::Value(value) = storage.slots()[2] { + assert_eq!(value, value_slot.into()); + } else { + panic!("Expected Value slot at index 2"); + } +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn test_storage_reconstruction_historical() { + let mut conn = create_db(); + + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + + // Block 1: Initial state with one value slot + let block_1 = BlockNumber::from(1); + create_block(&mut conn, block_1); + + let value_1 = [Felt::new(10), Felt::new(20), Felt::new(30), Felt::new(40)]; + queries::insert_account_storage_header( + &mut conn, + account_id, + block_1, + 0, + StorageSlotType::Value, + value_1.into(), + ) + .unwrap(); + + // Block 2: Update the value slot + let block_2 = BlockNumber::from(2); + create_block(&mut conn, block_2); + let value_2 = [Felt::new(50), Felt::new(60), Felt::new(70), Felt::new(80)]; + queries::insert_account_storage_header( + &mut conn, + account_id, + block_2, + 0, + StorageSlotType::Value, + value_2.into(), + ) + .unwrap(); + + // Reconstruct storage at block 1 + let storage_at_1 = + queries::select_account_storage_at_block(&mut conn, account_id, block_1).unwrap(); + assert_eq!(storage_at_1.slots().len(), 1); + if let miden_objects::account::StorageSlot::Value(value) = storage_at_1.slots()[0] { + assert_eq!(value, value_1.into()); + } else { + panic!("Expected Value slot"); + } + + // Reconstruct storage at block 2 + let storage_at_2 = + queries::select_account_storage_at_block(&mut conn, account_id, block_2).unwrap(); + assert_eq!(storage_at_2.slots().len(), 1); + if let miden_objects::account::StorageSlot::Value(value) = storage_at_2.slots()[0] { + assert_eq!(value, value_2.into()); + } else { + panic!("Expected Value slot"); + } + + // Latest should return block 2 value + let storage_latest = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + assert_eq!(storage_latest.slots().len(), 1); + if let miden_objects::account::StorageSlot::Value(value) = storage_latest.slots()[0] { + assert_eq!(value, value_2.into()); + } else { + panic!("Expected Value slot"); + } +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn test_storage_header_is_latest_flag() { + let mut conn = create_db(); + + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + let slot_index = 0u8; + + let value_1 = [Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]; + let value_2 = [Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]; + let value_3 = [Felt::new(9), Felt::new(10), Felt::new(11), Felt::new(12)]; + + // Create the blocks + create_block(&mut conn, BlockNumber::from(1)); + create_block(&mut conn, BlockNumber::from(2)); + create_block(&mut conn, BlockNumber::from(3)); + + // Insert at block 1 + queries::insert_account_storage_header( + &mut conn, + account_id, + BlockNumber::from(1), + slot_index, + StorageSlotType::Value, + value_1.into(), + ) + .unwrap(); + + // Insert at block 2 - should mark block 1 as not latest + queries::insert_account_storage_header( + &mut conn, + account_id, + BlockNumber::from(2), + slot_index, + StorageSlotType::Value, + value_2.into(), + ) + .unwrap(); + + // Insert at block 3 - should mark block 2 as not latest + queries::insert_account_storage_header( + &mut conn, + account_id, + BlockNumber::from(3), + slot_index, + StorageSlotType::Value, + value_3.into(), + ) + .unwrap(); + + // Query latest - should return block 3 + let storage_latest = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + assert_eq!(storage_latest.slots().len(), 1); + if let miden_objects::account::StorageSlot::Value(value) = storage_latest.slots()[0] { + assert_eq!(value, value_3.into()); + } else { + panic!("Expected Value slot with value_3"); + } + + // Verify historical queries still work + let storage_at_1 = + queries::select_account_storage_at_block(&mut conn, account_id, BlockNumber::from(1)) + .unwrap(); + if let miden_objects::account::StorageSlot::Value(value) = storage_at_1.slots()[0] { + assert_eq!(value, value_1.into()); + } else { + panic!("Expected Value slot with value_1"); + } +} + #[test] fn test_select_account_code_at_block() { let mut conn = create_db(); From 36470a5bf6bae1cb4cc3c8622c6c5f125a3e82f9 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 11 Dec 2025 16:01:13 +0100 Subject: [PATCH 34/95] improve --- .../db/migrations/2025062000000_setup/up.sql | 25 +- crates/store/src/db/mod.rs | 1 + .../store/src/db/models/queries/accounts.rs | 434 +++++++--------- .../src/db/models/queries/accounts/tests.rs | 478 ++++++++++++++++++ crates/store/src/db/schema.rs | 13 +- crates/store/src/db/tests.rs | 347 ------------- 6 files changed, 652 insertions(+), 646 deletions(-) create mode 100644 crates/store/src/db/models/queries/accounts/tests.rs diff --git a/crates/store/src/db/migrations/2025062000000_setup/up.sql b/crates/store/src/db/migrations/2025062000000_setup/up.sql index 0512499262..e02f23e0c8 100644 --- a/crates/store/src/db/migrations/2025062000000_setup/up.sql +++ b/crates/store/src/db/migrations/2025062000000_setup/up.sql @@ -19,14 +19,15 @@ CREATE TABLE accounts ( account_commitment BLOB NOT NULL, code_commitment BLOB, nonce INTEGER, + storage_header BLOB, -- Serialized AccountStorage from miden-objects is_latest BOOLEAN NOT NULL DEFAULT 0, -- Indicates if this is the latest state for this account_id PRIMARY KEY (account_id, block_num), CONSTRAINT all_null_or_none_null CHECK ( - (code_commitment IS NOT NULL AND nonce IS NOT NULL) + (code_commitment IS NOT NULL AND nonce IS NOT NULL AND storage_header IS NOT NULL) OR - (code_commitment IS NULL AND nonce IS NULL) + (code_commitment IS NULL AND nonce IS NULL AND storage_header IS NULL) ) ) WITHOUT ROWID; @@ -38,26 +39,6 @@ CREATE INDEX idx_accounts_block_num ON accounts(block_num); -- Index for joining with account_codes CREATE INDEX idx_accounts_code_commitment ON accounts(code_commitment) WHERE code_commitment IS NOT NULL; --- Table to store storage slot headers (slot types and commitments) -CREATE TABLE account_storage_headers ( - account_id BLOB NOT NULL, - block_num INTEGER NOT NULL, - slot_index INTEGER NOT NULL, - slot_type INTEGER NOT NULL, -- 0=Map, 1=Value (as per StorageSlotType) - slot_commitment BLOB NOT NULL, - is_latest BOOLEAN NOT NULL DEFAULT 0, - - PRIMARY KEY (account_id, block_num, slot_index), - CONSTRAINT slot_index_is_u8 CHECK (slot_index BETWEEN 0 AND 0xFF), - CONSTRAINT slot_type_in_enum CHECK (slot_type BETWEEN 0 AND 1), - FOREIGN KEY (account_id, block_num) REFERENCES accounts(account_id, block_num) ON DELETE CASCADE -) WITHOUT ROWID; - --- Index for joining with accounts table -CREATE INDEX idx_account_storage_headers_account_block ON account_storage_headers(account_id, block_num); --- Index for querying latest state -CREATE INDEX idx_account_storage_headers_latest ON account_storage_headers(account_id, is_latest) WHERE is_latest = 1; - CREATE TABLE notes ( committed_at INTEGER NOT NULL, -- Block number when the note was committed batch_index INTEGER NOT NULL, -- Index of batch in block, starting from 0 diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 5e32beafae..072497dfbd 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -424,6 +424,7 @@ impl Db { /// /// This method queries the decomposed storage tables and reconstructs the full /// `AccountStorage` with SMT backing for Map slots. + // TODO split querying the header from the content #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_account_storage_at_block( &self, diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 313ca0c362..b8180c3d62 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::ops::RangeInclusive; use diesel::prelude::{Queryable, QueryableByName}; @@ -27,6 +28,7 @@ use miden_objects::account::{ AccountHeader, AccountId, AccountStorage, + AccountStorageHeader, NonFungibleDeltaAction, StorageMap, StorageSlot, @@ -48,6 +50,9 @@ use crate::db::models::{serialize_vec, vec_raw_try_into}; use crate::db::{AccountVaultValue, schema}; use crate::errors::DatabaseError; +#[cfg(test)] +mod tests; + /// Select the latest account info by account id from the DB using the given /// [`SqliteConnection`]. /// @@ -506,7 +511,7 @@ pub(crate) fn reconstruct_storage_map_at_block( account_id: AccountId, block_num: BlockNumber, slot_index: u8, -) -> Result { +) -> Result { use schema::account_storage_map_values as t; // Check if the requested block exists (returns error if not) @@ -517,181 +522,156 @@ pub(crate) fn reconstruct_storage_map_at_block( let slot_sql = slot_to_raw_sql(slot_index); // Query all entries for this slot at or before the given block - let raw: Vec<(Vec, Vec)> = SelectDsl::select(t::table, (t::key, t::value)) - .filter( - t::account_id - .eq(&account_id_bytes) - .and(t::slot.eq(slot_sql)) - .and(t::block_num.le(block_num_sql)), - ) - .load(conn)?; + let raw: Vec<(i64, Vec, Vec)> = + SelectDsl::select(t::table, (t::block_num, t::key, t::value)) + .filter( + t::account_id + .eq(&account_id_bytes) + .and(t::slot.eq(slot_sql)) + .and(t::block_num.le(block_num_sql)), + ) + .load(conn)?; // Parse entries - let entries: Vec<(Word, Word)> = raw - .into_iter() - .map(|(k, v)| Ok((Word::read_from_bytes(&k)?, Word::read_from_bytes(&v)?))) - .collect::, DatabaseError>>()?; + let entries = Result::, DatabaseError>::from_iter( + raw.into_iter().map(|(n, key, value)| { + Ok(( + BlockNumber::from_raw_sql(n)?, + Word::read_from_bytes(&key)?, + Word::read_from_bytes(&value)?, + )) + }), + )?; + + // only keep the latest version of each key around + let mut key_map = HashMap::::new(); + for (block_num, key, value) in entries { + key_map + .entry(key) + .and_modify(|(existing_block, existing_value)| { + if block_num > *existing_block { + *existing_block = block_num; + *existing_value = value; + } + }) + .or_insert((block_num, value)); + } + + // Convert back to vec of (key, value) pairs + let entries: Vec<(Word, Word)> = + key_map.into_iter().map(|(key, (_block_num, value))| (key, value)).collect(); let entry_count = entries.len(); // StorageMap::with_entries internally uses an SMT which can be backed by SmtForest // The SMT is built with structural sharing for memory efficiency - miden_objects::account::StorageMap::with_entries(entries).map_err(|e| { + StorageMap::with_entries(entries).map_err(|e| { DatabaseError::DataCorrupted(format!( "Failed to create StorageMap from {entry_count} entries: {e}" )) }) } -/// Reconstruct `AccountStorage` from database tables for a specific account at a specific block -/// -/// This function queries the `account_storage_headers` table to get slot metadata and reconstructs -/// the `AccountStorage` without deserializing a blob. For Map slots, we only store the commitment -/// since the actual map data is in `account_storage_map_values`. -/// -/// # Returns -/// -/// The reconstructed `AccountStorage`, or an error if reconstruction fails. +/// Returns account storage header at a given block by reading from `accounts.storage_header` +/// and deserializing the storage header blob. pub(crate) fn select_account_storage_at_block( conn: &mut SqliteConnection, account_id: AccountId, block_num: BlockNumber, ) -> Result { - use schema::account_storage_headers as t; - - // Check if the requested block exists (returns error if not) block_exists(conn, block_num)?; let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); // Query storage headers for this account at this block - let headers: Vec = - SelectDsl::select(t::table, AccountStorageHeaderRaw::as_select()) - .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.eq(block_num_sql))) - .order(t::slot_index.asc()) - .load(conn)?; - - if headers.is_empty() { + let storage_header: Option> = + SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) + .filter(schema::accounts::account_id.eq(&account_id_bytes)) + .filter(schema::accounts::block_num.le(block_num_sql)) + .limit(1) + .first(conn)?; + + let Some(header) = storage_header else { // No storage headers means empty storage return Ok(AccountStorage::new(Vec::new())?); - } - - // Build slots from headers - let mut slots = Vec::with_capacity(headers.len()); - - for header in headers { - let slot_type = StorageSlotType::from_raw_sql(header.slot_type)?; + }; - let commitment = Word::read_from_bytes(&header.slot_commitment)?; + let header = AccountStorageHeader::read_from_bytes(&header)?; - let slot = match slot_type { + let mut slots = Vec::new(); + for (idx, (slot_ty, val)) in header.slots().enumerate() { + let slot = match slot_ty { StorageSlotType::Map => { - // For Map slots, we create an empty map - // The actual map data is queried separately when needed from - // account_storage_map_values - - // Create an empty storage map - let storage_map = StorageMap::new(); + let storage_map = + reconstruct_storage_map_at_block(conn, account_id, block_num, idx as u8)?; StorageSlot::Map(storage_map) }, - StorageSlotType::Value => { - // For Value slots, the commitment IS the value - StorageSlot::Value(commitment) - }, + StorageSlotType::Value => StorageSlot::Value(*val), }; - slots.push(slot); } Ok(AccountStorage::new(slots)?) } - -/// Select account storage headers at a specific block (lightweight query). -/// -/// Returns tuples of `(slot_index, slot_type, commitment)` without reconstructing full slots. -#[allow(dead_code)] // Helper for future SmtForest integration -pub(crate) fn select_account_storage_headers_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result, DatabaseError> { - use schema::account_storage_headers as t; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - - let headers: Vec = - SelectDsl::select(t::table, AccountStorageHeaderRaw::as_select()) - .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.le(block_num_sql))) - .order(t::slot_index.asc()) - .load(conn)?; - - headers - .into_iter() - .map(|h| { - let slot_index = raw_sql_to_slot(h.slot_index); - let slot_type = StorageSlotType::from_raw_sql(h.slot_type)?; - let commitment = Word::read_from_bytes(&h.slot_commitment)?; - Ok((slot_index, slot_type, commitment)) - }) - .collect() -} - -/// Reconstruct `AccountStorage` from the latest state in the database -/// -/// This queries only the latest storage headers (where `is_latest=true`) for faster reconstruction -/// Select the latest storage headers for an account -/// -/// This function queries the `account_storage_headers` table for the latest state of an account's -/// storage slots, using the `is_latest=true` flag for efficiency. -/// -/// # Returns -/// -/// The reconstructed `AccountStorage` from the latest storage headers. +/// Select latest account storage header by querying `accounts.storage_header` where +/// `is_latest=true`. pub(crate) fn select_latest_account_storage( conn: &mut SqliteConnection, account_id: AccountId, ) -> Result { - use schema::account_storage_headers as t; - let account_id_bytes = account_id.to_bytes(); - // Query latest storage headers for this account - let headers: Vec = - SelectDsl::select(t::table, AccountStorageHeaderRaw::as_select()) - .filter(t::account_id.eq(&account_id_bytes).and(t::is_latest.eq(true))) - .order(t::slot_index.asc()) - .load(conn)?; + // Query storage header for this account where is_latest = true + let storage_header: Option> = + SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) + .filter(schema::accounts::account_id.eq(&account_id_bytes)) + .filter(schema::accounts::is_latest.eq(true)) + .first(conn) + .optional()? + .flatten(); - if headers.is_empty() { + let Some(header) = storage_header else { // No storage headers means empty storage return Ok(AccountStorage::new(Vec::new())?); - } - - // Build slots from headers - let mut slots = Vec::with_capacity(headers.len()); + }; - for header in headers { - let slot_type = StorageSlotType::from_raw_sql(header.slot_type)?; - let slot_index = raw_sql_to_slot(header.slot_index); - let block_num = BlockNumber::from_raw_sql(header.block_num)?; - let commitment = Word::read_from_bytes(&header.slot_commitment)?; + let header = AccountStorageHeader::read_from_bytes(&header)?; - let slot = match slot_type { + let mut slots = Vec::new(); + for (idx, (slot_ty, val)) in header.slots().enumerate() { + let slot = match slot_ty { StorageSlotType::Map => { - // For Map slots, reconstruct the full SMT from database entries - // This allows serving proofs for any key in the map - let storage_map = - reconstruct_storage_map_at_block(conn, account_id, block_num, slot_index)?; + // For latest storage, we need to query all latest storage map values + // Note: we do not use `select_account_storage_at_block` here since we can use + // `is_latest=true` as a simplified filter. + use schema::account_storage_map_values as t; + + let raw: Vec<(Vec, Vec)> = SelectDsl::select(t::table, (t::key, t::value)) + .filter( + t::account_id + .eq(&account_id_bytes) + .and(t::slot.eq(slot_to_raw_sql(idx as u8))) + .and(t::is_latest.eq(true)), + ) + .load(conn)?; + + let entries = Result::, DatabaseError>::from_iter( + raw.into_iter().map(|(key, value)| { + Ok((Word::read_from_bytes(&key)?, Word::read_from_bytes(&value)?)) + }), + )?; + + let entry_count = entries.len(); + let storage_map = StorageMap::with_entries(entries).map_err(|e| { + DatabaseError::DataCorrupted(format!( + "Failed to create StorageMap from {entry_count} entries: {e}" + )) + })?; StorageSlot::Map(storage_map) }, - StorageSlotType::Value => { - // For Value slots, the commitment IS the value - StorageSlot::Value(commitment) - }, + StorageSlotType::Value => StorageSlot::Value(*val), }; - slots.push(slot); } @@ -719,19 +699,6 @@ impl TryFrom for AccountVaultValue { } } -#[derive(Debug, Clone, Queryable, Selectable)] -#[diesel(table_name = schema::account_storage_headers)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -#[allow(dead_code)] // Fields used by Diesel, not directly in Rust code -pub struct AccountStorageHeaderRaw { - pub account_id: Vec, - pub block_num: i64, - pub slot_index: i32, - pub slot_type: i32, - pub slot_commitment: Vec, - pub is_latest: bool, -} - #[derive(Debug, Clone, PartialEq, Eq, Selectable, Queryable, QueryableByName)] #[diesel(table_name = schema::accounts)] #[diesel(check_for_backend(Sqlite))] @@ -796,59 +763,6 @@ pub(crate) fn insert_account_vault_asset( }) } -/// Insert an account storage header into the DB using the given [`SqliteConnection`]. -/// -/// Sets `is_latest=true` for the new row and updates any existing -/// row with the same (`account_id`, `slot_index`) tuple to `is_latest=false`. -/// -/// # Returns -/// -/// The number of affected rows. -#[cfg(test)] -pub(crate) fn insert_account_storage_header( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, - slot_index: u8, - slot_type: StorageSlotType, - slot_commitment: Word, -) -> Result { - use schema::account_storage_headers as t; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - let slot_index_sql = slot_to_raw_sql(slot_index); - let slot_type_sql = slot_type.to_raw_sql(); - let slot_commitment_bytes = slot_commitment.to_bytes(); - - diesel::Connection::transaction(conn, |conn| { - // Update existing headers for this slot to set is_latest=false - let update_count = diesel::update(t::table) - .filter( - t::account_id - .eq(&account_id_bytes) - .and(t::slot_index.eq(slot_index_sql)) - .and(t::is_latest.eq(true)), - ) - .set(t::is_latest.eq(false)) - .execute(conn)?; - - // Insert the new latest row - let insert_count = diesel::insert_into(t::table) - .values(( - t::account_id.eq(&account_id_bytes), - t::block_num.eq(block_num_sql), - t::slot_index.eq(slot_index_sql), - t::slot_type.eq(slot_type_sql), - t::slot_commitment.eq(&slot_commitment_bytes), - t::is_latest.eq(true), - )) - .execute(conn)?; - - Ok(update_count + insert_count) - }) -} - /// Insert an account storage map value into the DB using the given [`SqliteConnection`]. /// /// Sets `is_latest=true` for the new row and updates any existing @@ -1090,6 +1004,7 @@ pub(crate) fn upsert_accounts( code_commitment: full_account .as_ref() .map(|account| account.code().commitment().to_bytes()), + storage_header: full_account.as_ref().map(|account| account.storage().to_bytes()), is_latest: true, }; @@ -1097,12 +1012,10 @@ pub(crate) fn upsert_accounts( .values(&account_value) .execute(conn)?; - // insert pending storage map entries for (acc_id, slot, key, value) in pending_storage_inserts { insert_account_storage_map_value(conn, acc_id, block_num, slot, key, value)?; } - // insert pending vault-asset entries for (acc_id, vault_key, update) in pending_asset_inserts { insert_account_vault_asset(conn, acc_id, block_num, vault_key, update)?; } @@ -1139,17 +1052,6 @@ pub(crate) struct AccountCodeRowInsert { pub(crate) code: Vec, } -#[derive(Insertable, AsChangeset, Debug, Clone)] -#[diesel(table_name = schema::account_storage_headers)] -pub(crate) struct AccountStorageHeaderInsert { - pub(crate) account_id: Vec, - pub(crate) block_num: i64, - pub(crate) slot_index: i32, - pub(crate) slot_type: i32, - pub(crate) slot_commitment: Vec, - pub(crate) is_latest: bool, -} - #[derive(Insertable, AsChangeset, Debug, Clone)] #[diesel(table_name = schema::accounts)] pub(crate) struct AccountRowInsert { @@ -1159,6 +1061,7 @@ pub(crate) struct AccountRowInsert { pub(crate) account_commitment: Vec, pub(crate) code_commitment: Option>, pub(crate) nonce: Option, + pub(crate) storage_header: Option>, pub(crate) is_latest: bool, } @@ -1206,7 +1109,7 @@ pub(crate) struct AccountStorageMapRowInsert { pub(crate) is_latest: bool, } -/// Queries vault assets (key, value) pairs at a specific block +/// Query vault assets at a specific block by finding the most recent update for each `vault_key`. pub(crate) fn select_account_vault_at_block( conn: &mut SqliteConnection, account_id: AccountId, @@ -1218,48 +1121,50 @@ pub(crate) fn select_account_vault_at_block( block_exists(conn, block_num)?; let account_id_bytes = account_id.to_bytes(); - let block_num_sql = i64::from(block_num.as_u32()); - let raw: Vec<(Vec, Option>)> = SelectDsl::select( + let block_num_sql = block_num.to_raw_sql(); + + // Since Diesel doesn't support composite keys in subqueries easily, we use a two-step approach: + // Step 1: Get max block_num for each vault_key + let latest_blocks_per_vault_key: Vec<(Vec, i64)> = QueryDsl::select( t::table .filter(t::account_id.eq(&account_id_bytes)) .filter(t::block_num.le(block_num_sql)) - .order(t::block_num.desc()) - .limit(1), - (t::vault_key, t::asset), + .group_by(t::vault_key), + (t::vault_key, diesel::dsl::max(t::block_num)), ) - .load(conn)?; + .load::<(Vec, Option)>(conn)? + .into_iter() + .filter_map(|(key, maybe_block)| maybe_block.map(|block| (key, block))) + .collect(); - let entries = raw - .into_iter() - .filter_map(|(key_bytes, maybe_asset_bytes)| { - let key = Word::read_from_bytes(&key_bytes).ok()?; - let asset_bytes = maybe_asset_bytes?; - let value = Word::read_from_bytes(&asset_bytes).ok()?; - Some((key, value)) - }) - .collect(); - - Ok(entries) -} + if latest_blocks_per_vault_key.is_empty() { + return Ok(Vec::new()); + } -/// Computes the storage commitment from a list of slot commitments. -/// -/// This replicates the logic from `AccountStorage::commitment()` which hashes all slot -/// commitments together. -/// -/// # Arguments -/// -/// * `slot_commitments` - Vector of slot commitment words -/// -/// # Returns -/// -/// The storage commitment as a `Word` -fn compute_storage_commitment(slot_commitments: &[Word]) -> Word { - use miden_objects::crypto::hash::rpo::Rpo256; + // Step 2: Fetch the full rows matching (vault_key, block_num) pairs + let mut entries = Vec::new(); + for (vault_key_bytes, max_block) in latest_blocks_per_vault_key { + let result: Option<(Vec, Option>)> = QueryDsl::select( + t::table.filter( + t::account_id + .eq(&account_id_bytes) + .and(t::vault_key.eq(&vault_key_bytes)) + .and(t::block_num.eq(max_block)), + ), + (t::vault_key, t::asset), + ) + .first(conn) + .optional()?; + if let Some((key_bytes, Some(asset_bytes))) = result { + entries + .push((Word::read_from_bytes(&key_bytes)?, Word::read_from_bytes(&asset_bytes)?)); + } + } - let elements: Vec = slot_commitments.iter().flat_map(|w| w.iter()).copied().collect(); + // Sort by vault_key for consistent ordering + entries.sort_by_key(|(key, _)| *key); - Rpo256::hash_elements(&elements) + Ok(entries) } /// Helper function to check if a block exists in the `block_headers` table. @@ -1339,12 +1244,18 @@ pub(crate) fn select_account_code_at_block( Ok(result) } +#[derive(Debug, Clone, Queryable)] +struct AccountHeaderDataRaw { + code_commitment: Option>, + nonce: Option, + storage_header: Option>, +} + /// Queries the account header for a specific account at a specific block number. /// /// This reconstructs the `AccountHeader` by joining multiple tables: -/// - `accounts` table for `account_id`, `nonce`, `code_commitment` +/// - `accounts` table for `account_id`, `nonce`, `code_commitment`, `storage_header` /// - `account_vault_headers` table for `vault_root` -/// - `account_storage_headers` table for storage slot commitments (to compute `storage_commitment`) /// /// Returns `None` if the account doesn't exist at that block. /// @@ -1364,27 +1275,33 @@ pub(crate) fn select_account_header_at_block( account_id: AccountId, block_num: BlockNumber, ) -> Result, DatabaseError> { - use schema::{account_storage_headers, account_vault_headers, accounts}; + use schema::{account_vault_headers, accounts}; - // Check if the requested block exists (returns error if not) block_exists(conn, block_num)?; let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); - let account_data: Option<(Option>, Option)> = SelectDsl::select( + + let account_data: Option = SelectDsl::select( accounts::table .filter(accounts::account_id.eq(&account_id_bytes)) .filter(accounts::block_num.le(block_num_sql)) .order(accounts::block_num.desc()) .limit(1), - (accounts::code_commitment, accounts::nonce), + (accounts::code_commitment, accounts::nonce, accounts::storage_header), ) .first(conn) .optional()?; - let Some((code_commitment_bytes, nonce_raw)) = account_data else { + let Some(AccountHeaderDataRaw { + code_commitment: code_commitment_bytes, + nonce: nonce_raw, + storage_header: storage_header_blob, + }) = account_data + else { return Ok(None); }; + let vault_root_bytes: Option> = SelectDsl::select( account_vault_headers::table .filter(account_vault_headers::account_id.eq(&account_id_bytes)) @@ -1396,26 +1313,13 @@ pub(crate) fn select_account_header_at_block( .first(conn) .optional()?; - let storage_slots: Vec<(i32, i32, Vec)> = SelectDsl::select( - account_storage_headers::table - .filter(account_storage_headers::account_id.eq(&account_id_bytes)) - .filter(account_storage_headers::block_num.le(block_num_sql)) - .order(account_storage_headers::block_num.desc()) - .limit(1), - ( - account_storage_headers::slot_index, - account_storage_headers::slot_type, - account_storage_headers::slot_commitment, - ), - ) - .load(conn)?; - - let slot_commitments: Vec = storage_slots - .into_iter() - .map(|(_slot_index, _slot_type, commitment_bytes)| Word::read_from_bytes(&commitment_bytes)) - .collect::, _>>()?; - - let storage_commitment = compute_storage_commitment(&slot_commitments); + let storage_commitment = match storage_header_blob { + Some(blob) => { + let storage = AccountStorage::read_from_bytes(&blob)?; + storage.commitment() + }, + None => Word::default(), + }; let code_commitment = code_commitment_bytes .map(|bytes| Word::read_from_bytes(&bytes)) diff --git a/crates/store/src/db/models/queries/accounts/tests.rs b/crates/store/src/db/models/queries/accounts/tests.rs new file mode 100644 index 0000000000..b68df7367e --- /dev/null +++ b/crates/store/src/db/models/queries/accounts/tests.rs @@ -0,0 +1,478 @@ +use assert_matches::assert_matches; +use diesel::{Connection, RunQueryDsl}; +use diesel_migrations::MigrationHarness; +use miden_lib::account::auth::AuthRpoFalcon512; +use miden_lib::transaction::TransactionKernel; +use miden_node_utils::fee::test_fee_params; +use miden_objects::account::auth::PublicKeyCommitment; +use miden_objects::account::{ + AccountBuilder, + AccountComponent, + AccountIdVersion, + AccountStorageMode, + AccountType, + StorageSlot, +}; +use miden_objects::{EMPTY_WORD, Word}; + +use super::*; +use crate::db::migrations::MIGRATIONS; + +fn setup_test_db() -> SqliteConnection { + let mut conn = + SqliteConnection::establish(":memory:").expect("Failed to create in-memory database"); + + conn.run_pending_migrations(MIGRATIONS).expect("Failed to run migrations"); + + conn +} + +fn create_test_account_with_storage() -> (Account, AccountId) { + // Create a simple public account with one value storage slot + let account_id = AccountId::dummy( + [1u8; 15], + AccountIdVersion::Version0, + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + ); + + let storage_value = Word::from([Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]); + let component_storage = vec![StorageSlot::Value(storage_value)]; + + let component = AccountComponent::compile( + "export.foo push.1 end", + TransactionKernel::assembler(), + component_storage, + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); + + let account = AccountBuilder::new([1u8; 32]) + .account_type(AccountType::RegularAccountImmutableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + (account, account_id) +} + +fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { + use miden_objects::block::BlockHeader; + + use crate::db::schema::block_headers; + + let block_header = BlockHeader::new( + 1_u8.into(), + Word::default(), + block_num, + Word::default(), + Word::default(), + Word::default(), + Word::default(), + Word::default(), + Word::default(), + Word::default(), + test_fee_params(), + 0_u8.into(), + ); + + diesel::insert_into(block_headers::table) + .values(( + block_headers::block_num.eq(i64::from(block_num.as_u32())), + block_headers::block_header.eq(block_header.to_bytes()), + )) + .execute(conn) + .expect("Failed to insert block header"); +} + +#[test] +fn test_upsert_accounts_inserts_storage_header() { + let mut conn = setup_test_db(); + let (account, account_id) = create_test_account_with_storage(); + + // Block 1 + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + let storage_commitment_original = account.storage().commitment(); + let storage_slots_len = account.storage().slots().len(); + let account_commitment = account.commitment(); + + // Create full state delta from the account + let delta = AccountDelta::try_from(account).unwrap(); + assert!(delta.is_full_state(), "Delta should be full state"); + + let account_update = + BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Delta(delta)); + + // Upsert account + let result = upsert_accounts(&mut conn, &[account_update], block_num); + assert!(result.is_ok(), "upsert_accounts failed: {:?}", result.err()); + assert_eq!(result.unwrap(), 1, "Expected 1 account to be inserted"); + + // Query storage header back + let queried_storage = select_latest_account_storage(&mut conn, account_id) + .expect("Failed to query storage header"); + + // Verify storage commitment matches + assert_eq!( + queried_storage.commitment(), + storage_commitment_original, + "Storage commitment mismatch" + ); + + // Verify number of slots matches + assert_eq!(queried_storage.slots().len(), storage_slots_len, "Storage slots count mismatch"); + + // Verify exactly 1 latest account with storage exists + let header_count: i64 = schema::accounts::table + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .filter(schema::accounts::storage_header.is_not_null()) + .count() + .get_result(&mut conn) + .expect("Failed to count accounts with storage"); + + assert_eq!(header_count, 1, "Expected exactly 1 latest account with storage"); +} + +#[test] +fn test_upsert_accounts_updates_is_latest_flag() { + let mut conn = setup_test_db(); + let (account, account_id) = create_test_account_with_storage(); + + // Block 1 and 2 + let block_num_1 = BlockNumber::from_epoch(0); + let block_num_2 = BlockNumber::from_epoch(1); + + insert_block_header(&mut conn, block_num_1); + insert_block_header(&mut conn, block_num_2); + + // Save storage commitment before moving account + let storage_commitment_1 = account.storage().commitment(); + let account_commitment_1 = account.commitment(); + + // First update with original account - full state delta + let delta_1 = AccountDelta::try_from(account).unwrap(); + + let account_update_1 = BlockAccountUpdate::new( + account_id, + account_commitment_1, + AccountUpdateDetails::Delta(delta_1), + ); + + upsert_accounts(&mut conn, &[account_update_1], block_num_1).expect("First upsert failed"); + + // Create modified account with different storage value + let storage_value_modified = + Word::from([Felt::new(10), Felt::new(20), Felt::new(30), Felt::new(40)]); + let component_storage_modified = vec![StorageSlot::Value(storage_value_modified)]; + + let component_2 = AccountComponent::compile( + "export.foo push.1 end", + TransactionKernel::assembler(), + component_storage_modified, + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); + + let account_2 = AccountBuilder::new([1u8; 32]) + .account_type(AccountType::RegularAccountImmutableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component_2) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + let storage_commitment_2 = account_2.storage().commitment(); + let account_commitment_2 = account_2.commitment(); + + // Second update with modified account - full state delta + let delta_2 = AccountDelta::try_from(account_2).unwrap(); + + let account_update_2 = BlockAccountUpdate::new( + account_id, + account_commitment_2, + AccountUpdateDetails::Delta(delta_2), + ); + + upsert_accounts(&mut conn, &[account_update_2], block_num_2).expect("Second upsert failed"); + + // Verify 2 total account rows exist (both historical records) + let total_accounts: i64 = schema::accounts::table + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .count() + .get_result(&mut conn) + .expect("Failed to count total accounts"); + + assert_eq!(total_accounts, 2, "Expected 2 total account records"); + + // Verify only 1 is marked as latest + let latest_accounts: i64 = schema::accounts::table + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .count() + .get_result(&mut conn) + .expect("Failed to count latest accounts"); + + assert_eq!(latest_accounts, 1, "Expected exactly 1 latest account"); + + // Verify latest storage matches second update + let latest_storage = select_latest_account_storage(&mut conn, account_id) + .expect("Failed to query latest storage"); + + assert_eq!( + latest_storage.commitment(), + storage_commitment_2, + "Latest storage should match second update" + ); + + // Verify historical query returns first update + let storage_at_block_1 = select_account_storage_at_block(&mut conn, account_id, block_num_1) + .expect("Failed to query storage at block 1"); + + assert_eq!( + storage_at_block_1.commitment(), + storage_commitment_1, + "Storage at block 1 should match first update" + ); +} + +#[test] +fn test_upsert_accounts_with_incremental_delta() { + use std::collections::BTreeMap; + + use miden_objects::account::delta::{AccountStorageDelta, AccountVaultDelta}; + + let mut conn = setup_test_db(); + let (account, account_id) = create_test_account_with_storage(); + + let block_num_1 = BlockNumber::from_epoch(0); + let block_num_2 = BlockNumber::from_epoch(1); + + insert_block_header(&mut conn, block_num_1); + insert_block_header(&mut conn, block_num_2); + + // First update with full state + let storage_commitment_1 = account.storage().commitment(); + let account_commitment_1 = account.commitment(); + let nonce_1 = account.nonce(); + let delta_1 = AccountDelta::try_from(account).unwrap(); + + let account_update_1 = BlockAccountUpdate::new( + account_id, + account_commitment_1, + AccountUpdateDetails::Delta(delta_1), + ); + + upsert_accounts(&mut conn, &[account_update_1], block_num_1).expect("First upsert failed"); + + // Create incremental delta (only modify storage value slot 1) + let new_storage_value = + Word::from([Felt::new(100), Felt::new(200), Felt::new(300), Felt::new(400)]); + + let mut storage_delta_values = BTreeMap::new(); + storage_delta_values.insert(1u8, new_storage_value); // Update slot 1 (component storage) + + let storage_delta = AccountStorageDelta::from_parts(storage_delta_values, BTreeMap::new()) + .expect("Failed to create storage delta"); + let incremental_delta = + AccountDelta::new(account_id, storage_delta, AccountVaultDelta::default(), nonce_1) + .expect("Failed to create incremental delta"); + + // Reconstruct expected account after delta + let account_after = reconstruct_full_account_from_db(&mut conn, account_id) + .expect("Failed to reconstruct account"); + let mut expected_account = account_after.clone(); + expected_account + .apply_delta(&incremental_delta) + .expect("Failed to apply delta to expected account"); + + let storage_commitment_2 = expected_account.storage().commitment(); + let account_commitment_2 = expected_account.commitment(); + + let account_update_2 = BlockAccountUpdate::new( + account_id, + account_commitment_2, + AccountUpdateDetails::Delta(incremental_delta), + ); + + upsert_accounts(&mut conn, &[account_update_2], block_num_2) + .expect("Second upsert with incremental delta failed"); + + // Verify latest storage matches expected state + let latest_storage = select_latest_account_storage(&mut conn, account_id) + .expect("Failed to query latest storage"); + + assert_eq!( + latest_storage.commitment(), + storage_commitment_2, + "Storage commitment should match after incremental delta" + ); + + // Verify historical storage is preserved + let storage_at_block_1 = select_account_storage_at_block(&mut conn, account_id, block_num_1) + .expect("Failed to query storage at block 1"); + + assert_eq!( + storage_at_block_1.commitment(), + storage_commitment_1, + "Historical storage should be unchanged" + ); +} + +#[test] +fn test_upsert_accounts_with_multiple_storage_slots() { + let mut conn = setup_test_db(); + + // Create account with 3 storage slots + let account_id = AccountId::dummy( + [2u8; 15], + AccountIdVersion::Version0, + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + ); + + let slot_value_1 = Word::from([Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]); + let slot_value_2 = Word::from([Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]); + let slot_value_3 = Word::from([Felt::new(9), Felt::new(10), Felt::new(11), Felt::new(12)]); + + let component_storage = vec![ + StorageSlot::Value(slot_value_1), + StorageSlot::Value(slot_value_2), + StorageSlot::Value(slot_value_3), + ]; + + let component = AccountComponent::compile( + "export.foo push.1 end", + TransactionKernel::assembler(), + component_storage, + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); + + let account = AccountBuilder::new([2u8; 32]) + .account_type(AccountType::RegularAccountImmutableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + let storage_commitment = account.storage().commitment(); + let account_commitment = account.commitment(); + let delta = AccountDelta::try_from(account).unwrap(); + + let account_update = + BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Delta(delta)); + + upsert_accounts(&mut conn, &[account_update], block_num) + .expect("Upsert with multiple storage slots failed"); + + // Query back and verify + let queried_storage = + select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + + assert_eq!(queried_storage.commitment(), storage_commitment, "Storage commitment mismatch"); + + // Note: Auth component adds 1 storage slot, so 3 component slots + 1 auth = 4 total + assert_eq!( + queried_storage.slots().len(), + 4, + "Expected 4 storage slots (3 component + 1 auth)" + ); + + // Verify individual slot values (skipping auth slot at index 0) + assert_matches!( + queried_storage.slots().get(1).expect("Slot 1 should exist"), + &StorageSlot::Value(v) if v == slot_value_1, + "Slot 1 value mismatch" + ); + assert_matches!( + queried_storage.slots().get(2).expect("Slot 2 should exist"), + &StorageSlot::Value(v) if v == slot_value_2, + "Slot 2 value mismatch" + ); + assert_matches!( + queried_storage.slots().get(3).expect("Slot 3 should exist"), + &StorageSlot::Value(v) if v == slot_value_3, + "Slot 3 value mismatch" + ); +} + +#[test] +fn test_upsert_accounts_with_empty_storage() { + let mut conn = setup_test_db(); + + // Create account with no storage slots + let account_id = AccountId::dummy( + [3u8; 15], + AccountIdVersion::Version0, + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + ); + + let component = AccountComponent::compile( + "export.foo push.1 end", + TransactionKernel::assembler(), + vec![], // Empty storage + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); + + let account = AccountBuilder::new([3u8; 32]) + .account_type(AccountType::RegularAccountImmutableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + let storage_commitment = account.storage().commitment(); + let account_commitment = account.commitment(); + let delta = AccountDelta::try_from(account).unwrap(); + + let account_update = + BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Delta(delta)); + + upsert_accounts(&mut conn, &[account_update], block_num) + .expect("Upsert with empty storage failed"); + + // Query back and verify + let queried_storage = + select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + + assert_eq!( + queried_storage.commitment(), + storage_commitment, + "Storage commitment mismatch for empty storage" + ); + + // Note: Auth component adds 1 storage slot, so even "empty" accounts have 1 slot + assert_eq!(queried_storage.slots().len(), 1, "Expected 1 storage slot (auth component)"); + + // Verify the storage header blob exists in database + let storage_header_exists: Option = SelectDsl::select( + schema::accounts::table + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)), + schema::accounts::storage_header.is_not_null(), + ) + .first(&mut conn) + .optional() + .expect("Failed to check storage header existence"); + + assert_eq!( + storage_header_exists, + Some(true), + "Storage header blob should exist even for empty storage" + ); +} diff --git a/crates/store/src/db/schema.rs b/crates/store/src/db/schema.rs index 4929d3e106..18d557bdde 100644 --- a/crates/store/src/db/schema.rs +++ b/crates/store/src/db/schema.rs @@ -1,16 +1,5 @@ // @generated automatically by Diesel CLI. -diesel::table! { - account_storage_headers (account_id, block_num, slot_index) { - account_id -> Binary, - block_num -> BigInt, - slot_index -> Integer, - slot_type -> Integer, - slot_commitment -> Binary, - is_latest -> Bool, - } -} - diesel::table! { account_storage_map_values (account_id, block_num, slot, key) { account_id -> Binary, @@ -48,6 +37,7 @@ diesel::table! { account_commitment -> Binary, code_commitment -> Nullable, nonce -> Nullable, + storage_header -> Nullable, block_num -> BigInt, is_latest -> Bool, } @@ -130,7 +120,6 @@ diesel::joinable!(transactions -> block_headers (block_num)); diesel::allow_tables_to_appear_in_same_query!( account_codes, - account_storage_headers, account_storage_map_values, accounts, account_vault_assets, diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 12263c0f5b..e4910b1a43 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -20,14 +20,11 @@ use miden_objects::account::{ AccountDelta, AccountId, AccountIdVersion, - AccountStorage, AccountStorageDelta, AccountStorageMode, AccountType, AccountVaultDelta, - StorageMap, StorageSlot, - StorageSlotType, }; use miden_objects::asset::{Asset, AssetVaultKey, FungibleAsset}; use miden_objects::block::{ @@ -1541,350 +1538,6 @@ fn mock_account_code_and_storage( // STORAGE RECONSTRUCTION TESTS // ================================================================================================ -#[test] -#[miden_node_test_macro::enable_logging] -fn test_storage_reconstruction_latest_state() { - let mut conn = create_db(); - - // Create an account with storage slots - let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - let block_num = BlockNumber::from(1); - - // Create the block before inserting storage data - create_block(&mut conn, block_num); - - // Create test storage with Value and Map slots - let value_slot = StorageSlot::Value(num_to_word(42)); - let mut storage_map = StorageMap::new(); - let _ = storage_map.insert(num_to_word(1), num_to_word(100)); - let _ = storage_map.insert(num_to_word(2), num_to_word(200)); - let map_slot = StorageSlot::Map(storage_map.clone()); - - let _storage = AccountStorage::new(vec![value_slot, map_slot]).unwrap(); - - // Insert storage headers for both slots - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num, - 0, // slot_index - miden_objects::account::StorageSlotType::Value, - num_to_word(42), - ) - .unwrap(); - - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num, - 1, // slot_index - miden_objects::account::StorageSlotType::Map, - storage_map.root(), - ) - .unwrap(); - - // Insert map values - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block_num, - 1, // slot - num_to_word(1), // key - num_to_word(100), // value - ) - .unwrap(); - - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block_num, - 1, // slot - num_to_word(2), // key - num_to_word(200), // value - ) - .unwrap(); - - // Reconstruct storage from latest state - let reconstructed_storage = - queries::select_latest_account_storage(&mut conn, account_id).unwrap(); - - // Verify reconstructed storage - assert_eq!(reconstructed_storage.slots().len(), 2); - - // Check Value slot - match &reconstructed_storage.slots()[0] { - StorageSlot::Value(v) => assert_eq!(*v, num_to_word(42)), - StorageSlot::Map(_) => panic!("Expected Value slot"), - } - - // Check Map slot (commitment should match) - match &reconstructed_storage.slots()[1] { - StorageSlot::Map(_) => { - // The map should be reconstructed (empty but with correct slot type) - // Actual values would need to be queried separately from account_storage_map_values - }, - StorageSlot::Value(_) => panic!("Expected Map slot"), - } -} - -#[test] -#[miden_node_test_macro::enable_logging] -fn test_storage_reconstruction_historical_state() { - let mut conn = create_db(); - - let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - - // Block 1: Initial storage - let block_num_1 = BlockNumber::from(1); - create_block(&mut conn, block_num_1); - - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num_1, - 0, - miden_objects::account::StorageSlotType::Value, - num_to_word(10), - ) - .unwrap(); - - // Block 2: Updated storage - let block_num_2 = BlockNumber::from(2); - create_block(&mut conn, block_num_2); - - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num_2, - 0, - miden_objects::account::StorageSlotType::Value, - num_to_word(20), - ) - .unwrap(); - - // Reconstruct storage at block 1 - let storage_block_1 = - queries::select_account_storage_at_block(&mut conn, account_id, block_num_1).unwrap(); - match &storage_block_1.slots()[0] { - StorageSlot::Value(v) => assert_eq!(*v, num_to_word(10)), - StorageSlot::Map(_) => panic!("Expected Value slot"), - } - - // Reconstruct storage at block 2 - let storage_block_2 = - queries::select_account_storage_at_block(&mut conn, account_id, block_num_2).unwrap(); - match &storage_block_2.slots()[0] { - StorageSlot::Value(v) => assert_eq!(*v, num_to_word(20)), - StorageSlot::Map(_) => panic!("Expected Value slot"), - } - - // Reconstruct latest storage (should match block 2) - let storage_latest = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); - match &storage_latest.slots()[0] { - StorageSlot::Value(v) => assert_eq!(*v, num_to_word(20)), - StorageSlot::Map(_) => panic!("Expected Value slot"), - } -} - -#[test] -#[miden_node_test_macro::enable_logging] -fn test_storage_reconstruction_latest() { - let mut conn = create_db(); - - let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - let block_num = BlockNumber::from(1); - - // Create the block - create_block(&mut conn, block_num); - - // Insert storage headers: 2 Map slots and 1 Value slot - let map_commitment_1 = [Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]; - let map_commitment_2 = [Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]; - let value_slot = [Felt::new(100), Felt::new(200), Felt::new(300), Felt::new(400)]; - - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num, - 0, // slot 0: Map - StorageSlotType::Map, - map_commitment_1.into(), - ) - .unwrap(); - - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num, - 1, // slot 1: Map - StorageSlotType::Map, - map_commitment_2.into(), - ) - .unwrap(); - - queries::insert_account_storage_header( - &mut conn, - account_id, - block_num, - 2, // slot 2: Value - StorageSlotType::Value, - value_slot.into(), - ) - .unwrap(); - - // Reconstruct storage from headers - let storage = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); - - // Verify we have 3 slots - assert_eq!(storage.slots().len(), 3); - - // Verify slot types - assert!(matches!(storage.slots()[0], miden_objects::account::StorageSlot::Map(_))); - assert!(matches!(storage.slots()[1], miden_objects::account::StorageSlot::Map(_))); - - if let miden_objects::account::StorageSlot::Value(value) = storage.slots()[2] { - assert_eq!(value, value_slot.into()); - } else { - panic!("Expected Value slot at index 2"); - } -} - -#[test] -#[miden_node_test_macro::enable_logging] -fn test_storage_reconstruction_historical() { - let mut conn = create_db(); - - let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - - // Block 1: Initial state with one value slot - let block_1 = BlockNumber::from(1); - create_block(&mut conn, block_1); - - let value_1 = [Felt::new(10), Felt::new(20), Felt::new(30), Felt::new(40)]; - queries::insert_account_storage_header( - &mut conn, - account_id, - block_1, - 0, - StorageSlotType::Value, - value_1.into(), - ) - .unwrap(); - - // Block 2: Update the value slot - let block_2 = BlockNumber::from(2); - create_block(&mut conn, block_2); - let value_2 = [Felt::new(50), Felt::new(60), Felt::new(70), Felt::new(80)]; - queries::insert_account_storage_header( - &mut conn, - account_id, - block_2, - 0, - StorageSlotType::Value, - value_2.into(), - ) - .unwrap(); - - // Reconstruct storage at block 1 - let storage_at_1 = - queries::select_account_storage_at_block(&mut conn, account_id, block_1).unwrap(); - assert_eq!(storage_at_1.slots().len(), 1); - if let miden_objects::account::StorageSlot::Value(value) = storage_at_1.slots()[0] { - assert_eq!(value, value_1.into()); - } else { - panic!("Expected Value slot"); - } - - // Reconstruct storage at block 2 - let storage_at_2 = - queries::select_account_storage_at_block(&mut conn, account_id, block_2).unwrap(); - assert_eq!(storage_at_2.slots().len(), 1); - if let miden_objects::account::StorageSlot::Value(value) = storage_at_2.slots()[0] { - assert_eq!(value, value_2.into()); - } else { - panic!("Expected Value slot"); - } - - // Latest should return block 2 value - let storage_latest = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); - assert_eq!(storage_latest.slots().len(), 1); - if let miden_objects::account::StorageSlot::Value(value) = storage_latest.slots()[0] { - assert_eq!(value, value_2.into()); - } else { - panic!("Expected Value slot"); - } -} - -#[test] -#[miden_node_test_macro::enable_logging] -fn test_storage_header_is_latest_flag() { - let mut conn = create_db(); - - let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - let slot_index = 0u8; - - let value_1 = [Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]; - let value_2 = [Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]; - let value_3 = [Felt::new(9), Felt::new(10), Felt::new(11), Felt::new(12)]; - - // Create the blocks - create_block(&mut conn, BlockNumber::from(1)); - create_block(&mut conn, BlockNumber::from(2)); - create_block(&mut conn, BlockNumber::from(3)); - - // Insert at block 1 - queries::insert_account_storage_header( - &mut conn, - account_id, - BlockNumber::from(1), - slot_index, - StorageSlotType::Value, - value_1.into(), - ) - .unwrap(); - - // Insert at block 2 - should mark block 1 as not latest - queries::insert_account_storage_header( - &mut conn, - account_id, - BlockNumber::from(2), - slot_index, - StorageSlotType::Value, - value_2.into(), - ) - .unwrap(); - - // Insert at block 3 - should mark block 2 as not latest - queries::insert_account_storage_header( - &mut conn, - account_id, - BlockNumber::from(3), - slot_index, - StorageSlotType::Value, - value_3.into(), - ) - .unwrap(); - - // Query latest - should return block 3 - let storage_latest = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); - assert_eq!(storage_latest.slots().len(), 1); - if let miden_objects::account::StorageSlot::Value(value) = storage_latest.slots()[0] { - assert_eq!(value, value_3.into()); - } else { - panic!("Expected Value slot with value_3"); - } - - // Verify historical queries still work - let storage_at_1 = - queries::select_account_storage_at_block(&mut conn, account_id, BlockNumber::from(1)) - .unwrap(); - if let miden_objects::account::StorageSlot::Value(value) = storage_at_1.slots()[0] { - assert_eq!(value, value_1.into()); - } else { - panic!("Expected Value slot with value_1"); - } -} - #[test] fn test_select_account_code_at_block() { let mut conn = create_db(); From 928fdb4c3506430f48ce20082ccaf7d3f9545b6f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 15:14:43 +0100 Subject: [PATCH 35/95] yes --- .../store/src/db/models/queries/accounts.rs | 44 ------------------- crates/store/src/state.rs | 30 +++++++++++++ 2 files changed, 30 insertions(+), 44 deletions(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index b8180c3d62..c686baf094 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -514,9 +514,6 @@ pub(crate) fn reconstruct_storage_map_at_block( ) -> Result { use schema::account_storage_map_values as t; - // Check if the requested block exists (returns error if not) - block_exists(conn, block_num)?; - let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); let slot_sql = slot_to_raw_sql(slot_index); @@ -579,8 +576,6 @@ pub(crate) fn select_account_storage_at_block( account_id: AccountId, block_num: BlockNumber, ) -> Result { - block_exists(conn, block_num)?; - let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); @@ -1117,9 +1112,6 @@ pub(crate) fn select_account_vault_at_block( ) -> Result, DatabaseError> { use schema::account_vault_assets as t; - // Check if the requested block exists (returns error if not) - block_exists(conn, block_num)?; - let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); @@ -1167,37 +1159,6 @@ pub(crate) fn select_account_vault_at_block( Ok(entries) } -/// Helper function to check if a block exists in the `block_headers` table. -/// -/// This should be called by all `_at_block` query functions to ensure that -/// queries are only performed against blocks that have been produced. -/// -/// # Arguments -/// -/// * `conn` - Database connection -/// * `block_num` - The block number to check -/// -/// # Returns -/// -/// * `Ok(())` - If the block exists -/// * `Err(DatabaseError::BlockNotFound)` - If the block doesn't exist -/// * `Err(DatabaseError)` - If there's a database error -fn block_exists(conn: &mut SqliteConnection, block_num: BlockNumber) -> Result<(), DatabaseError> { - use schema::block_headers; - - let count: i64 = SelectDsl::select( - block_headers::table.filter(block_headers::block_num.eq(block_num.to_raw_sql())), - diesel::dsl::count(block_headers::block_num), - ) - .first(conn)?; - - if count > 0 { - Ok(()) - } else { - Err(DatabaseError::BlockNotFound(block_num)) - } -} - /// Queries the account code for a specific account at a specific block number. /// /// Returns `None` if: @@ -1222,9 +1183,6 @@ pub(crate) fn select_account_code_at_block( ) -> Result>, DatabaseError> { use schema::{account_codes, accounts}; - // Check if the requested block exists (returns error if not) - block_exists(conn, block_num)?; - let account_id_bytes = account_id.to_bytes(); let block_num_sql = i64::from(block_num.as_u32()); // Query the accounts table to get the code_commitment at the specified block or earlier @@ -1277,8 +1235,6 @@ pub(crate) fn select_account_header_at_block( ) -> Result, DatabaseError> { use schema::{account_vault_headers, accounts}; - block_exists(conn, block_num)?; - let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 1e32d95145..125e34f3a8 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -1137,11 +1137,18 @@ impl State { } /// Reconstructs account storage at a specific block + /// + /// # Errors + /// + /// Returns an error if the block doesn't exist or if there's a database error. pub async fn get_account_storage_at_block( &self, account_id: AccountId, block_num: BlockNumber, ) -> Result { + // Validate block exists in the blockchain before querying the database + self.validate_block_exists(block_num).await?; + self.db.select_account_storage_at_block(account_id, block_num).await } @@ -1233,6 +1240,9 @@ impl State { return Err(DatabaseError::AccountNotPublic(account_id)); } + // Validate block exists in the blockchain before querying the database + self.validate_block_exists(block_num).await?; + let account_header = self .db .select_account_header_at_block(account_id, block_num) @@ -1315,6 +1325,26 @@ impl State { self.inner.read().await.latest_block_num() } + /// Validates that a block exists in the blockchain + /// + /// # Attention + /// + /// Acquires a *read lock** on `self.inner`. + /// + /// # Errors + /// + /// Returns `DatabaseError::BlockNotFound` if the block doesn't exist in the blockchain. + async fn validate_block_exists(&self, block_num: BlockNumber) -> Result<(), DatabaseError> { + let inner = self.inner.read().await; + let latest_block_num = inner.latest_block_num(); + + if block_num > latest_block_num { + return Err(DatabaseError::BlockNotFound(block_num)); + } + + Ok(()) + } + /// Runs database optimization. pub async fn optimize_db(&self) -> Result<(), DatabaseError> { self.db.optimize().await From 5de3936a0691cfe7784c57943ae2a11393873259 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 17:23:50 +0100 Subject: [PATCH 36/95] tuple ticks --- crates/proto/src/domain/account.rs | 96 +++++++++++++---------- crates/store/src/inner_forest.rs | 121 ++++++++++++++++++++++++++++- crates/store/src/state.rs | 75 ++---------------- 3 files changed, 180 insertions(+), 112 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index a9e84c6348..54f5f39421 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -349,29 +349,34 @@ impl From for proto::account::AccountStorageHeader { /// Account vault assets /// -/// Represents a list of assets, if the number of assets is reasonably small, which -/// is currently set to 1000 for no particular reason. +/// Represents the assets in an account's vault, with proper handling for vaults +/// containing many assets. /// -/// When an account contains a large number of assets, including all assets -/// in a single RPC response would create performance issues on client and server as -/// and consume quite a bit of bandwidth, besides requiring additional memory on -/// possibly low powered clients. +/// When an account contains a large number of assets (> 1000), including all assets +/// in a single RPC response would create performance issues on client and server, +/// consume significant bandwidth, and require additional memory on possibly low-powered clients. /// -/// Hence `too_many_assets` is returned, which is indicating to the client to use the dedicated -/// `SyncAccountVault` RPC endpoint and do incremental retrieval +/// In such cases, the `LimitExceeded` variant indicates to the client to use the dedicated +/// `SyncAccountVault` RPC endpoint for incremental retrieval. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct AccountVaultDetails { - /// Flag indicating whether the vault has too many assets to return inline. - /// If `true`, clients must use `SyncAccountVault` endpoint instead. - pub too_many_assets: bool, +pub enum AccountVaultAssets { + /// The vault has too many assets to return inline. + /// Clients must use `SyncAccountVault` endpoint instead. + LimitExceeded, + + /// The assets in the vault (up to `MAX_RETURN_ENTRIES`). + Assets(Vec), +} - /// The assets in the vault. Empty if `too_many_assets` is `true`. - pub assets: Vec, +/// Account vault details - wrapper for backwards compatibility with protobuf +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccountVaultDetails { + pub assets: AccountVaultAssets, } impl AccountVaultDetails { /// Maximum number of vault entries that can be returned in a single response. - /// Accounts with more assets will have `too_many_assets = true` and empty `assets`. + /// Accounts with more assets will have `LimitExceeded` variant. const MAX_RETURN_ENTRIES: usize = 1000; pub fn new(vault: &AssetVault) -> Self { @@ -379,16 +384,14 @@ impl AccountVaultDetails { Self::too_many() } else { Self { - too_many_assets: false, - assets: Vec::from_iter(vault.assets()), + assets: AccountVaultAssets::Assets(Vec::from_iter(vault.assets())), } } } pub fn empty() -> Self { Self { - too_many_assets: false, - assets: Vec::new(), + assets: AccountVaultAssets::Assets(Vec::new()), } } @@ -399,9 +402,7 @@ impl AccountVaultDetails { /// /// The entries are `(vault_key, asset)` pairs where `asset` is a Word representation. pub fn from_entries(entries: Vec<(Word, Word)>) -> Result { - let too_many_assets = entries.len() > Self::MAX_RETURN_ENTRIES; - - if too_many_assets { + if entries.len() > Self::MAX_RETURN_ENTRIES { return Ok(Self::too_many()); } @@ -410,13 +411,14 @@ impl AccountVaultDetails { .map(|(_key, asset_word)| Asset::try_from(asset_word)) .collect::, _>>()?; - Ok(Self { too_many_assets: false, assets }) + Ok(Self { + assets: AccountVaultAssets::Assets(assets), + }) } fn too_many() -> Self { Self { - too_many_assets: true, - assets: Vec::new(), + assets: AccountVaultAssets::LimitExceeded, } } } @@ -427,27 +429,41 @@ impl TryFrom for AccountVaultDetails { fn try_from(value: proto::rpc::AccountVaultDetails) -> Result { let proto::rpc::AccountVaultDetails { too_many_assets, assets } = value; - let assets = - Result::, ConversionError>::from_iter(assets.into_iter().map(|asset| { - let asset = asset - .asset - .ok_or(proto::primitives::Asset::missing_field(stringify!(asset)))?; - let asset = Word::try_from(asset)?; - Asset::try_from(asset).map_err(ConversionError::AssetError) - }))?; - Ok(Self { too_many_assets, assets }) + if too_many_assets { + Ok(Self { + assets: AccountVaultAssets::LimitExceeded, + }) + } else { + let parsed_assets = + Result::, ConversionError>::from_iter(assets.into_iter().map(|asset| { + let asset = asset + .asset + .ok_or(proto::primitives::Asset::missing_field(stringify!(asset)))?; + let asset = Word::try_from(asset)?; + Asset::try_from(asset).map_err(ConversionError::AssetError) + }))?; + Ok(Self { + assets: AccountVaultAssets::Assets(parsed_assets), + }) + } } } impl From for proto::rpc::AccountVaultDetails { fn from(value: AccountVaultDetails) -> Self { - let AccountVaultDetails { too_many_assets, assets } = value; + let AccountVaultDetails { assets } = value; - Self { - too_many_assets, - assets: Vec::from_iter(assets.into_iter().map(|asset| proto::primitives::Asset { - asset: Some(proto::primitives::Digest::from(Word::from(asset))), - })), + match assets { + AccountVaultAssets::LimitExceeded => Self { + too_many_assets: true, + assets: Vec::new(), + }, + AccountVaultAssets::Assets(assets) => Self { + too_many_assets: false, + assets: Vec::from_iter(assets.into_iter().map(|asset| proto::primitives::Asset { + asset: Some(proto::primitives::Digest::from(Word::from(asset))), + })), + }, } } } diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index a57db588e7..cefc1f2a82 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -1,9 +1,9 @@ use std::collections::BTreeMap; -use miden_objects::Word; -use miden_objects::account::AccountId; +use miden_objects::account::{AccountId, AccountStorage, StorageSlot}; use miden_objects::block::BlockNumber; use miden_objects::crypto::merkle::SmtForest; +use miden_objects::{EMPTY_WORD, Word}; /// Container for forest-related state that needs to be updated atomically. pub(crate) struct InnerForest { @@ -13,11 +13,11 @@ pub(crate) struct InnerForest { /// Maps (`account_id`, `slot_index`, `block_num`) to SMT root. /// Populated during block import for all storage map slots. - pub(crate) storage_roots: BTreeMap<(AccountId, u8, BlockNumber), Word>, + storage_roots: BTreeMap<(AccountId, u8, BlockNumber), Word>, /// Maps (`account_id`, `block_num`) to vault SMT root. /// Tracks asset vault versions across all blocks with structural sharing. - pub(crate) vault_roots: BTreeMap<(AccountId, BlockNumber), Word>, + vault_roots: BTreeMap<(AccountId, BlockNumber), Word>, } impl InnerForest { @@ -28,4 +28,117 @@ impl InnerForest { vault_roots: BTreeMap::new(), } } + + /// Extracts map-type storage slots and their entries from account storage data. + /// + /// This is a helper method to prepare data for populating the forest with storage maps. + /// It iterates through all accounts' storage slots and collects only the map-type slots + /// with their entries. + /// + /// # Arguments + /// + /// * `account_storages` - Slice of `(account_id, storage)` tuples from database + /// + /// # Returns + /// + /// Vec of `(account_id, slot_index, entries)` tuples ready for forest population + #[allow(clippy::type_complexity)] + pub(crate) fn extract_map_slots_from_storage( + account_storages: &[(AccountId, AccountStorage)], + ) -> Vec<(AccountId, u8, Vec<(&Word, &Word)>)> { + let mut map_slots = Vec::new(); + + for (account_id, storage) in account_storages { + for (slot_idx, slot) in storage.slots().iter().enumerate() { + if let StorageSlot::Map(storage_map) = slot { + let entries = Vec::from_iter(storage_map.entries()); + map_slots.push((*account_id, slot_idx as u8, entries)); + } + } + } + + tracing::debug!(target: crate::COMPONENT, num_map_slots = map_slots.len()); + map_slots + } + + /// Populates the forest with storage map SMTs for the given slots. + /// + /// This method builds SMTs from the provided entries and tracks their roots, + /// enabling efficient historical queries with structural sharing. + /// + /// # Arguments + /// + /// * `map_slots` - Vec of `(account_id, slot_index, entries)` tuples + /// * `block_num` - Block number for which these SMTs are being created + #[allow(clippy::type_complexity)] + pub(crate) fn populate_storage_maps( + &mut self, + map_slots: Vec<(AccountId, u8, Vec<(&Word, &Word)>)>, + block_num: BlockNumber, + ) { + let prev_block_num = block_num.parent().unwrap_or_default(); + + for (account_id, slot_idx, entries) in map_slots { + // Get previous root for structural sharing + let prev_root = self + .storage_roots + .get(&(account_id, slot_idx, prev_block_num)) + .copied() + .unwrap_or(EMPTY_WORD); + + // Build new SMT from entries + let updated_root = self + .storage_forest + .batch_insert(prev_root, entries.into_iter().map(|(k, v)| (*k, *v))) + .expect("Forest insertion should always succeed with valid entries"); + + // Track the new root + self.storage_roots.insert((account_id, slot_idx, block_num), updated_root); + } + + tracing::debug!( + target: crate::COMPONENT, + total_tracked_roots = self.storage_roots.len(), + "Updated storage map roots" + ); + } + + /// Populates the forest with vault SMTs for the given accounts. + /// + /// This method builds vault SMTs from the provided asset entries and tracks their roots, + /// enabling efficient historical queries with structural sharing. + /// + /// # Arguments + /// + /// * `vault_entries` - Vec of `(account_id, entries)` tuples where entries are (key, value) pairs + /// * `block_num` - Block number for which these vault SMTs are being created + pub(crate) fn populate_vaults( + &mut self, + vault_entries: Vec<(AccountId, Vec<(Word, Word)>)>, + block_num: BlockNumber, + ) { + let prev_block_num = block_num.parent().unwrap_or_default(); + + for (account_id, entries) in vault_entries { + let prev_root = self + .vault_roots + .get(&(account_id, prev_block_num)) + .copied() + .unwrap_or(EMPTY_WORD); + + let updated_root = self + .storage_forest + .batch_insert(prev_root, entries) + .expect("Database is consistent and always allows constructing a smt or forest"); + + // Track the new vault root + self.vault_roots.insert((account_id, block_num), updated_root); + } + + tracing::debug!( + target: crate::COMPONENT, + total_vault_roots = self.vault_roots.len(), + "Updated vault roots" + ); + } } diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 125e34f3a8..97de2c115a 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -50,7 +50,7 @@ use miden_objects::crypto::merkle::{ use miden_objects::note::{NoteDetails, NoteId, NoteScript, Nullifier}; use miden_objects::transaction::{OutputNote, PartialBlockchain}; use miden_objects::utils::Serializable; -use miden_objects::{AccountError, EMPTY_WORD, Word}; +use miden_objects::{AccountError, Word}; use tokio::sync::{Mutex, RwLock, oneshot}; use tracing::{info, info_span, instrument}; @@ -512,8 +512,8 @@ impl State { let account_storages = self.query_account_storages_from_db(changed_account_ids, block_num).await?; - // Step 2: Extract map slots and their entries - let map_slots_to_populate = Self::extract_map_slots_from_storage(&account_storages); + // Step 2: Extract map slots and their entries using InnerForest helper + let map_slots_to_populate = InnerForest::extract_map_slots_from_storage(&account_storages); // Step 3: Update the forest with new SMTs self.populate_forest_with_storage_maps(map_slots_to_populate, block_num).await?; @@ -538,27 +538,6 @@ impl State { Ok(account_storages) } - /// Extracts map-type storage slots and their entries from account storage data - #[instrument(target = COMPONENT, skip_all, fields(num_accounts = account_storages.len()))] - #[allow(clippy::type_complexity)] - fn extract_map_slots_from_storage<'a>( - account_storages: &'a [(AccountId, miden_objects::account::AccountStorage)], - ) -> Vec<(AccountId, u8, Vec<(&'a Word, &'a Word)>)> { - let mut map_slots = Vec::new(); - - for (account_id, storage) in account_storages { - for (slot_idx, slot) in storage.slots().iter().enumerate() { - if let StorageSlot::Map(storage_map) = slot { - let entries = Vec::from_iter(storage_map.entries()); - map_slots.push((*account_id, slot_idx as u8, entries)); - } - } - } - - tracing::debug!(target: COMPONENT, num_map_slots = map_slots.len()); - map_slots - } - /// Populates the forest with storage map SMTs for the given slots #[instrument(target = COMPONENT, skip_all, fields(num_slots = map_slots.len()))] #[allow(clippy::type_complexity)] @@ -573,29 +552,10 @@ impl State { // Acquire write lock once for all updates let mut forest_guard = self.forest.write().await; - let prev_block_num = block_num.parent().unwrap_or_default(); - for (account_id, slot_idx, entries) in map_slots { - // Get previous root for structural sharing - let prev_root = forest_guard - .storage_roots - .get(&(account_id, slot_idx, prev_block_num)) - .copied() - .unwrap_or(EMPTY_WORD); - - // Build new SMT from entries - let updated_root = forest_guard - .storage_forest - .batch_insert(prev_root, entries.into_iter().map(|(k, v)| (*k, *v))) - .expect("Forest insertion should always succeed with valid entries"); - - // Track the new root - forest_guard - .storage_roots - .insert((account_id, slot_idx, block_num), updated_root); - } + // Delegate to InnerForest for the actual population logic + forest_guard.populate_storage_maps(map_slots, block_num); - tracing::debug!(target: COMPONENT, total_tracked_roots = forest_guard.storage_roots.len()); Ok(()) } @@ -620,31 +580,10 @@ impl State { return Ok(()); } - // Acquire a single write lock on the forest for the entire update operation. - // Since apply_block() is already serialized by the `writer` Mutex, holding this lock - // for the entire duration is acceptable and simplifies the code. + // Acquire write lock once for the entire update operation and delegate to InnerForest let mut forest_guard = self.forest.write().await; + forest_guard.populate_vaults(vault_entries_to_populate, block_num); - let prev_block_num = block_num.parent().unwrap_or_default(); - - // Process each vault: get previous root, build new SMT, track new root - for (account_id, entries) in vault_entries_to_populate { - let prev_root = forest_guard - .vault_roots - .get(&(account_id, prev_block_num)) - .copied() - .unwrap_or(EMPTY_WORD); - - let updated_root = forest_guard - .storage_forest - .batch_insert(prev_root, entries) - .expect("Database is consistent and always allows constructing a smt or forest"); - - // Track the new vault root - forest_guard.vault_roots.insert((account_id, block_num), updated_root); - } - - tracing::debug!(target: COMPONENT, total_vault_roots = forest_guard.vault_roots.len()); Ok(()) } From ca5ef9a36558d1559e47e61e3cba67ef1cd368c3 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 18:36:35 +0100 Subject: [PATCH 37/95] docs --- .../db/migrations/2025062000000_setup/up.sql | 21 ++-------- .../store/src/db/models/queries/accounts.rs | 40 +++++++++---------- crates/store/src/db/schema.rs | 11 +---- crates/store/src/db/tests.rs | 40 +++++++++---------- crates/store/src/state.rs | 2 +- crates/utils/src/limiter.rs | 3 +- 6 files changed, 44 insertions(+), 73 deletions(-) diff --git a/crates/store/src/db/migrations/2025062000000_setup/up.sql b/crates/store/src/db/migrations/2025062000000_setup/up.sql index 200a3a63a3..3f7449292d 100644 --- a/crates/store/src/db/migrations/2025062000000_setup/up.sql +++ b/crates/store/src/db/migrations/2025062000000_setup/up.sql @@ -20,14 +20,15 @@ CREATE TABLE accounts ( code_commitment BLOB, nonce INTEGER, storage_header BLOB, -- Serialized AccountStorage from miden-objects + vault_root BLOB, -- Vault root commitment is_latest BOOLEAN NOT NULL DEFAULT 0, -- Indicates if this is the latest state for this account_id PRIMARY KEY (account_id, block_num), CONSTRAINT all_null_or_none_null CHECK ( - (code_commitment IS NOT NULL AND nonce IS NOT NULL AND storage_header IS NOT NULL) + (code_commitment IS NOT NULL AND nonce IS NOT NULL AND storage_header IS NOT NULL AND vault_root IS NOT NULL) OR - (code_commitment IS NULL AND nonce IS NULL AND storage_header IS NULL) + (code_commitment IS NULL AND nonce IS NULL AND storage_header IS NULL AND vault_root IS NULL) ) ) WITHOUT ROWID; @@ -120,22 +121,6 @@ CREATE INDEX idx_vault_assets_account_block ON account_vault_assets(account_id, -- Index for querying latest assets CREATE INDEX idx_vault_assets_latest ON account_vault_assets(account_id, is_latest) WHERE is_latest = 1; --- Table to store vault headers (vault root commitments) -CREATE TABLE account_vault_headers ( - account_id BLOB NOT NULL, - block_num INTEGER NOT NULL, - vault_root BLOB NOT NULL, - is_latest BOOLEAN NOT NULL DEFAULT 0, - - PRIMARY KEY (account_id, block_num), - FOREIGN KEY (account_id, block_num) REFERENCES accounts(account_id, block_num) ON DELETE CASCADE -) WITHOUT ROWID; - --- Index for joining with accounts table -CREATE INDEX idx_account_vault_headers_account_block ON account_vault_headers(account_id, block_num); --- Index for querying latest state -CREATE INDEX idx_account_vault_headers_latest ON account_vault_headers(account_id, is_latest) WHERE is_latest = 1; - CREATE TABLE nullifiers ( nullifier BLOB NOT NULL, nullifier_prefix INTEGER NOT NULL, diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 76185a2bb3..d931addb65 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -885,6 +885,7 @@ pub(crate) fn upsert_accounts( .as_ref() .map(|account| account.code().commitment().to_bytes()), storage_header: full_account.as_ref().map(|account| account.storage().to_bytes()), + vault_root: full_account.as_ref().map(|account| account.vault().root().to_bytes()), is_latest: true, }; @@ -943,6 +944,7 @@ pub(crate) struct AccountRowInsert { pub(crate) code_commitment: Option>, pub(crate) nonce: Option, pub(crate) storage_header: Option>, + pub(crate) vault_root: Option>, pub(crate) is_latest: bool, } @@ -1097,9 +1099,8 @@ struct AccountHeaderDataRaw { /// Queries the account header for a specific account at a specific block number. /// -/// This reconstructs the `AccountHeader` by joining multiple tables: -/// - `accounts` table for `account_id`, `nonce`, `code_commitment`, `storage_header` -/// - `account_vault_headers` table for `vault_root` +/// This reconstructs the `AccountHeader` by reading from the `accounts` table: +/// - `account_id`, `nonce`, `code_commitment`, `storage_header`, `vault_root` /// /// Returns `None` if the account doesn't exist at that block. /// @@ -1119,42 +1120,37 @@ pub(crate) fn select_account_header_at_block( account_id: AccountId, block_num: BlockNumber, ) -> Result, DatabaseError> { - use schema::{account_vault_headers, accounts}; + use schema::accounts; let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); - let account_data: Option = SelectDsl::select( + let account_data: Option<(AccountHeaderDataRaw, Option>)> = SelectDsl::select( accounts::table .filter(accounts::account_id.eq(&account_id_bytes)) .filter(accounts::block_num.le(block_num_sql)) .order(accounts::block_num.desc()) .limit(1), - (accounts::code_commitment, accounts::nonce, accounts::storage_header), + ( + (accounts::code_commitment, accounts::nonce, accounts::storage_header), + accounts::vault_root, + ), ) .first(conn) .optional()?; - let Some(AccountHeaderDataRaw { - code_commitment: code_commitment_bytes, - nonce: nonce_raw, - storage_header: storage_header_blob, - }) = account_data + let Some(( + AccountHeaderDataRaw { + code_commitment: code_commitment_bytes, + nonce: nonce_raw, + storage_header: storage_header_blob, + }, + vault_root_bytes, + )) = account_data else { return Ok(None); }; - let vault_root_bytes: Option> = SelectDsl::select( - account_vault_headers::table - .filter(account_vault_headers::account_id.eq(&account_id_bytes)) - .filter(account_vault_headers::block_num.le(block_num_sql)) - .order(account_vault_headers::block_num.desc()) - .limit(1), - account_vault_headers::vault_root, - ) - .first(conn) - .optional()?; - let storage_commitment = match storage_header_blob { Some(blob) => { let storage = AccountStorage::read_from_bytes(&blob)?; diff --git a/crates/store/src/db/schema.rs b/crates/store/src/db/schema.rs index e767f56793..90c48380d1 100644 --- a/crates/store/src/db/schema.rs +++ b/crates/store/src/db/schema.rs @@ -21,15 +21,6 @@ diesel::table! { } } -diesel::table! { - account_vault_headers (account_id, block_num) { - account_id -> Binary, - block_num -> BigInt, - vault_root -> Binary, - is_latest -> Bool, - } -} - diesel::table! { accounts (account_id, block_num) { account_id -> Binary, @@ -38,6 +29,7 @@ diesel::table! { code_commitment -> Nullable, nonce -> Nullable, storage_header -> Nullable, + vault_root -> Nullable, block_num -> BigInt, is_latest -> Bool, } @@ -123,7 +115,6 @@ diesel::allow_tables_to_appear_in_same_query!( account_storage_map_values, accounts, account_vault_assets, - account_vault_headers, block_headers, note_scripts, notes, diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 7920877dbd..a75d020492 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -463,27 +463,25 @@ fn sql_unconsumed_network_notes() { create_block(&mut conn, 1.into()); // Create an unconsumed note in each block. - let notes = (0..2) - .map(|i: u32| { - let note = NoteRecord { - block_num: 0.into(), // Created on same block. - note_index: BlockNoteIndex::new(0, i as usize).unwrap(), - note_id: num_to_word(i.into()), - note_commitment: num_to_word(i.into()), - metadata: NoteMetadata::new( - account_note.0, - NoteType::Public, - NoteTag::from_account_id(account_note.0), - NoteExecutionHint::none(), - Felt::default(), - ) - .unwrap(), - details: None, - inclusion_path: SparseMerklePath::default(), - }; - (note, Some(num_to_nullifier(i.into()))) - }) - .collect::>(); + let notes = Vec::from_iter((0..2).map(|i: u32| { + let note = NoteRecord { + block_num: 0.into(), // Created on same block. + note_index: BlockNoteIndex::new(0, i as usize).unwrap(), + note_id: num_to_word(i.into()), + note_commitment: num_to_word(i.into()), + metadata: NoteMetadata::new( + account_note.0, + NoteType::Public, + NoteTag::from_account_id(account_note.0), + NoteExecutionHint::none(), + Felt::default(), + ) + .unwrap(), + details: None, + inclusion_path: SparseMerklePath::default(), + }; + (note, Some(num_to_nullifier(i.into()))) + })); queries::insert_scripts(&mut conn, notes.iter().map(|(note, _)| note)).unwrap(); queries::insert_notes(&mut conn, ¬es).unwrap(); diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 98d90d7f7b..a428a58849 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -1376,7 +1376,7 @@ async fn load_account_tree( db: &mut Db, block_number: BlockNumber, ) -> Result, StateInitializationError> { - let account_data = db.select_all_account_commitments().await?.into_iter().collect::>(); + let account_data = Vec::from_iter(db.select_all_account_commitments().await?); let smt_entries = account_data .into_iter() diff --git a/crates/utils/src/limiter.rs b/crates/utils/src/limiter.rs index cf43407174..3b3c478821 100644 --- a/crates/utils/src/limiter.rs +++ b/crates/utils/src/limiter.rs @@ -11,7 +11,8 @@ //! //! Add new limits here so callers share the same values and rationale. -const GENERAL_REQUEST_LIMIT: usize = 1000; +/// Basic request limit. +pub const GENERAL_REQUEST_LIMIT: usize = 1000; #[allow(missing_docs)] #[derive(Debug, thiserror::Error)] From 17fd95b3fd721eeb5cbb107bc6ea82f63df2d010 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 19:42:41 +0100 Subject: [PATCH 38/95] from_iter --- crates/proto/src/domain/account.rs | 16 ++- crates/store/src/db/mod.rs | 3 +- .../store/src/db/models/queries/accounts.rs | 36 +++--- crates/store/src/db/tests.rs | 2 +- crates/store/src/server/api.rs | 2 +- crates/store/src/server/ntx_builder.rs | 4 +- crates/store/src/server/rpc_api.rs | 111 ++++++++---------- crates/store/src/state.rs | 13 +- 8 files changed, 88 insertions(+), 99 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index c2587025f2..0053c86748 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -335,14 +335,13 @@ impl From for proto::account::AccountHeader { impl From for proto::account::AccountStorageHeader { fn from(value: AccountStorageHeader) -> Self { - let slots = value - .slots() - .map(|slot_header| proto::account::account_storage_header::StorageSlot { + let slots = Vec::from_iter(value.slots().map(|slot_header| { + proto::account::account_storage_header::StorageSlot { slot_name: slot_header.name().to_string(), slot_type: storage_slot_type_to_raw(slot_header.slot_type()), commitment: Some(proto::primitives::Digest::from(slot_header.value())), - }) - .collect(); + } + })); Self { slots } } @@ -407,10 +406,9 @@ impl AccountVaultDetails { return Ok(Self::too_many()); } - let assets = entries - .into_iter() - .map(|(_key, asset_word)| Asset::try_from(asset_word)) - .collect::, _>>()?; + let assets = Result::, _>::from_iter( + entries.into_iter().map(|(_key, asset_word)| Asset::try_from(asset_word)), + )?; Ok(Self { assets: AccountVaultAssets::Assets(assets), diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index c106446cd7..78c366ba12 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -112,8 +112,7 @@ impl TransactionRecord { self, note_records: Vec, ) -> proto::rpc::TransactionRecord { - let output_notes: Vec = - note_records.into_iter().map(Into::into).collect(); + let output_notes = Vec::from_iter(note_records.into_iter().map(Into::into)); proto::rpc::TransactionRecord { header: Some(proto::transaction::TransactionHeader { diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index d931addb65..2c931f27f5 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -338,14 +338,11 @@ pub(crate) fn select_all_accounts( let summaries: Vec = vec_raw_try_into(raw).unwrap(); // Backfill account details from database - let account_infos = summaries - .into_iter() - .map(|summary| { - let account_id = summary.account_id; - let details = reconstruct_full_account_from_db(conn, account_id).ok(); - AccountInfo { summary, details } - }) - .collect(); + let account_infos = Vec::from_iter(summaries.into_iter().map(|summary| { + let account_id = summary.account_id; + let details = reconstruct_full_account_from_db(conn, account_id).ok(); + AccountInfo { summary, details } + })); Ok(account_infos) } @@ -1005,17 +1002,18 @@ pub(crate) fn select_account_vault_at_block( // Since Diesel doesn't support composite keys in subqueries easily, we use a two-step approach: // Step 1: Get max block_num for each vault_key - let latest_blocks_per_vault_key: Vec<(Vec, i64)> = QueryDsl::select( - t::table - .filter(t::account_id.eq(&account_id_bytes)) - .filter(t::block_num.le(block_num_sql)) - .group_by(t::vault_key), - (t::vault_key, diesel::dsl::max(t::block_num)), - ) - .load::<(Vec, Option)>(conn)? - .into_iter() - .filter_map(|(key, maybe_block)| maybe_block.map(|block| (key, block))) - .collect(); + let latest_blocks_per_vault_key = Vec::from_iter( + QueryDsl::select( + t::table + .filter(t::account_id.eq(&account_id_bytes)) + .filter(t::block_num.le(block_num_sql)) + .group_by(t::vault_key), + (t::vault_key, diesel::dsl::max(t::block_num)), + ) + .load::<(Vec, Option)>(conn)? + .into_iter() + .filter_map(|(key, maybe_block)| maybe_block.map(|block| (key, block))), + ); if latest_blocks_per_vault_key.is_empty() { return Ok(Vec::new()); diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index a75d020492..fb5043a4db 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1162,7 +1162,7 @@ fn sql_account_storage_map_values_insertion() { let mut map1 = StorageMapDelta::default(); map1.insert(key1, value1); map1.insert(key2, value2); - let maps1: BTreeMap<_, _> = [(slot_name.clone(), map1)].into_iter().collect(); + let maps1 = BTreeMap::from_iter([(slot_name.clone(), map1)]); let storage1 = AccountStorageDelta::from_parts(BTreeMap::new(), maps1).unwrap(); let delta1 = AccountDelta::new(account_id, storage1, AccountVaultDelta::default(), Felt::ONE).unwrap(); diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index b266feb59e..65bd07d9f7 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -163,5 +163,5 @@ pub fn validate_note_commitments(notes: &[proto::primitives::Digest]) -> Result< #[instrument(level = "debug",target = COMPONENT, skip_all)] pub fn read_block_numbers(block_numbers: &[u32]) -> BTreeSet { - block_numbers.iter().map(|raw_number| BlockNumber::from(*raw_number)).collect() + BTreeSet::from_iter(block_numbers.iter().map(|raw_number| BlockNumber::from(*raw_number))) } diff --git a/crates/store/src/server/ntx_builder.rs b/crates/store/src/server/ntx_builder.rs index 91bc5a6480..54422dc64e 100644 --- a/crates/store/src/server/ntx_builder.rs +++ b/crates/store/src/server/ntx_builder.rs @@ -165,8 +165,8 @@ impl ntx_builder_server::NtxBuilder for StoreApi { ) -> Result, Status> { let account_ids = self.state.get_all_network_accounts().await.map_err(internal_error)?; - let account_ids: Vec = - account_ids.into_iter().map(Into::into).collect(); + let account_ids = + Vec::from_iter(account_ids.into_iter().map(Into::into)); Ok(Response::new(proto::store::NetworkAccountIdList { account_ids })) } diff --git a/crates/store/src/server/rpc_api.rs b/crates/store/src/server/rpc_api.rs index 748cd07703..de1da89c01 100644 --- a/crates/store/src/server/rpc_api.rs +++ b/crates/store/src/server/rpc_api.rs @@ -91,7 +91,7 @@ impl rpc_server::Rpc for StoreApi { let proofs = self.state.check_nullifiers(&nullifiers).await; Ok(Response::new(proto::rpc::CheckNullifiersResponse { - proofs: convert(proofs).collect(), + proofs: Vec::from_iter(convert(proofs)), })) } @@ -128,13 +128,12 @@ impl rpc_server::Rpc for StoreApi { .await .map_err(SyncNullifiersError::from)?; - let nullifiers = nullifiers - .into_iter() - .map(|nullifier_info| proto::rpc::sync_nullifiers_response::NullifierUpdate { + let nullifiers = Vec::from_iter(nullifiers.into_iter().map(|nullifier_info| { + proto::rpc::sync_nullifiers_response::NullifierUpdate { nullifier: Some(nullifier_info.nullifier.into()), block_num: nullifier_info.block_num.as_u32(), - }) - .collect(); + } + })); Ok(Response::new(proto::rpc::SyncNullifiersResponse { pagination_info: Some(proto::rpc::PaginationInfo { @@ -170,27 +169,24 @@ impl rpc_server::Rpc for StoreApi { .await .map_err(internal_error)?; - let accounts = state - .account_updates - .into_iter() - .map(|account_info| proto::account::AccountSummary { + let accounts = Vec::from_iter(state.account_updates.into_iter().map(|account_info| { + proto::account::AccountSummary { account_id: Some(account_info.account_id.into()), account_commitment: Some(account_info.account_commitment.into()), block_num: account_info.block_num.as_u32(), - }) - .collect(); - - let transactions = state - .transactions - .into_iter() - .map(|transaction_summary| proto::transaction::TransactionSummary { - account_id: Some(transaction_summary.account_id.into()), - block_num: transaction_summary.block_num.as_u32(), - transaction_id: Some(transaction_summary.transaction_id.into()), - }) - .collect(); + } + })); + + let transactions = + Vec::from_iter(state.transactions.into_iter().map(|transaction_summary| { + proto::transaction::TransactionSummary { + account_id: Some(transaction_summary.account_id.into()), + block_num: transaction_summary.block_num.as_u32(), + transaction_id: Some(transaction_summary.transaction_id.into()), + } + })); - let notes = state.notes.into_iter().map(Into::into).collect(); + let notes = Vec::from_iter(state.notes.into_iter().map(Into::into)); Ok(Response::new(proto::rpc::SyncStateResponse { chain_tip: self.state.latest_block_num().await.as_u32(), @@ -229,7 +225,7 @@ impl rpc_server::Rpc for StoreApi { let (state, mmr_proof, last_block_included) = self.state.sync_notes(request.note_tags, block_range).await?; - let notes = state.notes.into_iter().map(Into::into).collect(); + let notes = Vec::from_iter(state.notes.into_iter().map(Into::into)); Ok(Response::new(proto::rpc::SyncNotesResponse { pagination_info: Some(proto::rpc::PaginationInfo { @@ -268,16 +264,16 @@ impl rpc_server::Rpc for StoreApi { let note_ids: Vec = convert_digests_to_words::(note_ids)?; - let note_ids: Vec = note_ids.into_iter().map(NoteId::from_raw).collect(); + let note_ids = Vec::from_iter(note_ids.into_iter().map(NoteId::from_raw)); - let notes = self - .state - .get_notes_by_id(note_ids) - .await - .map_err(GetNotesByIdError::from)? - .into_iter() - .map(Into::into) - .collect(); + let notes = Vec::from_iter( + self.state + .get_notes_by_id(note_ids) + .await + .map_err(GetNotesByIdError::from)? + .into_iter() + .map(Into::into), + ); Ok(Response::new(proto::note::CommittedNoteList { notes })) } @@ -387,17 +383,14 @@ impl rpc_server::Rpc for StoreApi { .await .map_err(SyncAccountVaultError::from)?; - let updates = updates - .into_iter() - .map(|update| { - let vault_key: Word = update.vault_key.into(); - proto::rpc::AccountVaultUpdate { - vault_key: Some(vault_key.into()), - asset: update.asset.map(Into::into), - block_num: update.block_num.as_u32(), - } - }) - .collect(); + let updates = Vec::from_iter(updates.into_iter().map(|update| { + let vault_key: Word = update.vault_key.into(); + proto::rpc::AccountVaultUpdate { + vault_key: Some(vault_key.into()), + asset: update.asset.map(Into::into), + block_num: update.block_num.as_u32(), + } + })); Ok(Response::new(proto::rpc::SyncAccountVaultResponse { pagination_info: Some(proto::rpc::PaginationInfo { @@ -445,16 +438,14 @@ impl rpc_server::Rpc for StoreApi { .await .map_err(SyncStorageMapsError::from)?; - let updates = storage_maps_page - .values - .into_iter() - .map(|map_value| proto::rpc::StorageMapUpdate { + let updates = Vec::from_iter(storage_maps_page.values.into_iter().map(|map_value| { + proto::rpc::StorageMapUpdate { slot_name: map_value.slot_name.to_string(), key: Some(map_value.key.into()), value: Some(map_value.value.into()), block_num: map_value.block_num.as_u32(), - }) - .collect(); + } + })); Ok(Response::new(proto::rpc::SyncStorageMapsResponse { pagination_info: Some(proto::rpc::PaginationInfo { @@ -562,21 +553,23 @@ impl rpc_server::Rpc for StoreApi { .map_err(SyncTransactionsError::from)?; // Create a map from note ID to note record for efficient lookup - let note_map: std::collections::HashMap<_, _> = all_note_records - .into_iter() - .map(|note_record| (note_record.note_id, note_record)) - .collect(); + let note_map: std::collections::HashMap<_, _> = std::collections::HashMap::from_iter( + all_note_records + .into_iter() + .map(|note_record| (note_record.note_id, note_record)), + ); // Convert database TransactionRecord to proto TransactionRecord let mut transactions = Vec::with_capacity(transaction_records_db.len()); for tx_header in transaction_records_db { // Get note records for this transaction's output notes - let note_records: Vec<_> = tx_header - .output_notes - .iter() - .filter_map(|note_id| note_map.get(¬e_id.as_word()).cloned()) - .collect(); + let note_records = Vec::from_iter( + tx_header + .output_notes + .iter() + .filter_map(|note_id| note_map.get(¬e_id.as_word()).cloned()), + ); // Convert to proto using the helper method let proto_record = tx_header.into_proto_with_note_records(note_records); diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index a428a58849..d803c2767e 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -171,7 +171,7 @@ impl State { // necessary in theory let acc_account_ids = me.db.select_all_account_commitments().await?; let acc_account_ids = - acc_account_ids.into_iter().map(|(account_id, _)| account_id).collect(); + Vec::from_iter(acc_account_ids.into_iter().map(|(account_id, _)| account_id)); me.update_storage_forest_from_db(acc_account_ids, latest_block_num) .await .map_err(|e| { @@ -337,11 +337,12 @@ impl State { }; // build note tree - let note_tree_entries: Vec<_> = block - .body() - .output_notes() - .map(|(note_index, note)| (note_index, note.id(), *note.metadata())) - .collect(); + let note_tree_entries = Vec::from_iter( + block + .body() + .output_notes() + .map(|(note_index, note)| (note_index, note.id(), *note.metadata())), + ); let note_tree = miden_objects::block::BlockNoteTree::with_entries(note_tree_entries.iter().copied()) .map_err(|e| InvalidBlockError::FailedToBuildNoteTree(e.to_string()))?; From 22f3ca9e6443bc4787964c7e9ffe221a45374fd2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 19:59:55 +0100 Subject: [PATCH 39/95] simplify --- crates/proto/src/domain/account.rs | 48 ++++++++---------------------- 1 file changed, 12 insertions(+), 36 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 0053c86748..60e3c4f842 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -347,7 +347,7 @@ impl From for proto::account::AccountStorageHeader { } } -/// Account vault assets +/// Account vault details /// /// Represents the assets in an account's vault, with proper handling for vaults /// containing many assets. @@ -359,7 +359,7 @@ impl From for proto::account::AccountStorageHeader { /// In such cases, the `LimitExceeded` variant indicates to the client to use the dedicated /// `SyncAccountVault` RPC endpoint for incremental retrieval. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum AccountVaultAssets { +pub enum AccountVaultDetails { /// The vault has too many assets to return inline. /// Clients must use `SyncAccountVault` endpoint instead. LimitExceeded, @@ -368,12 +368,6 @@ pub enum AccountVaultAssets { Assets(Vec), } -/// Account vault details - wrapper for backwards compatibility with protobuf -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AccountVaultDetails { - pub assets: AccountVaultAssets, -} - impl AccountVaultDetails { /// Maximum number of vault entries that can be returned in a single response. /// Accounts with more assets will have `LimitExceeded` variant. @@ -381,18 +375,14 @@ impl AccountVaultDetails { pub fn new(vault: &AssetVault) -> Self { if vault.assets().nth(Self::MAX_RETURN_ENTRIES).is_some() { - Self::too_many() + Self::LimitExceeded } else { - Self { - assets: AccountVaultAssets::Assets(Vec::from_iter(vault.assets())), - } + Self::Assets(Vec::from_iter(vault.assets())) } } pub fn empty() -> Self { - Self { - assets: AccountVaultAssets::Assets(Vec::new()), - } + Self::Assets(Vec::new()) } /// Creates `AccountVaultDetails` from vault entries (key-value pairs). @@ -403,22 +393,14 @@ impl AccountVaultDetails { /// The entries are `(vault_key, asset)` pairs where `asset` is a Word representation. pub fn from_entries(entries: Vec<(Word, Word)>) -> Result { if entries.len() > Self::MAX_RETURN_ENTRIES { - return Ok(Self::too_many()); + return Ok(Self::LimitExceeded); } let assets = Result::, _>::from_iter( entries.into_iter().map(|(_key, asset_word)| Asset::try_from(asset_word)), )?; - Ok(Self { - assets: AccountVaultAssets::Assets(assets), - }) - } - - fn too_many() -> Self { - Self { - assets: AccountVaultAssets::LimitExceeded, - } + Ok(Self::Assets(assets)) } } @@ -429,9 +411,7 @@ impl TryFrom for AccountVaultDetails { let proto::rpc::AccountVaultDetails { too_many_assets, assets } = value; if too_many_assets { - Ok(Self { - assets: AccountVaultAssets::LimitExceeded, - }) + Ok(Self::LimitExceeded) } else { let parsed_assets = Result::, ConversionError>::from_iter(assets.into_iter().map(|asset| { @@ -441,23 +421,19 @@ impl TryFrom for AccountVaultDetails { let asset = Word::try_from(asset)?; Asset::try_from(asset).map_err(ConversionError::AssetError) }))?; - Ok(Self { - assets: AccountVaultAssets::Assets(parsed_assets), - }) + Ok(Self::Assets(parsed_assets)) } } } impl From for proto::rpc::AccountVaultDetails { fn from(value: AccountVaultDetails) -> Self { - let AccountVaultDetails { assets } = value; - - match assets { - AccountVaultAssets::LimitExceeded => Self { + match value { + AccountVaultDetails::LimitExceeded => Self { too_many_assets: true, assets: Vec::new(), }, - AccountVaultAssets::Assets(assets) => Self { + AccountVaultDetails::Assets(assets) => Self { too_many_assets: false, assets: Vec::from_iter(assets.into_iter().map(|asset| proto::primitives::Asset { asset: Some(proto::primitives::Digest::from(Word::from(asset))), From 3ee1884662e9c4259b52c3fcda3c958f2dc0cc40 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 20:18:34 +0100 Subject: [PATCH 40/95] docs --- crates/proto/src/domain/account.rs | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 60e3c4f842..0d7bfbde2d 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -349,15 +349,9 @@ impl From for proto::account::AccountStorageHeader { /// Account vault details /// -/// Represents the assets in an account's vault, with proper handling for vaults -/// containing many assets. -/// /// When an account contains a large number of assets (> 1000), including all assets -/// in a single RPC response would create performance issues on client and server, -/// consume significant bandwidth, and require additional memory on possibly low-powered clients. -/// -/// In such cases, the `LimitExceeded` variant indicates to the client to use the dedicated -/// `SyncAccountVault` RPC endpoint for incremental retrieval. +/// in a single RPC response creates performance issues. In such cases, the `LimitExceeded` +/// variant indicates to the client to use the `SyncAccountVault` endpoint instead. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AccountVaultDetails { /// The vault has too many assets to return inline. @@ -445,18 +439,9 @@ impl From for proto::rpc::AccountVaultDetails { /// Details about an account storage map slot, including overflow handling. /// -/// ## Rationale for "Too Many Entries" Flag -/// -/// Similar to `AccountVaultDetails`, when a storage map contains many entries (> 1000), -/// returning all entries in a single RPC response creates performance issues: -/// - Large serialization/deserialization costs -/// - Network bandwidth saturation -/// - Client memory pressure -/// -/// When `too_many_entries` is `true`: -/// - The `map_entries` field is empty (no data included) -/// - Clients should use the dedicated `SyncStorageMaps` RPC endpoint -/// - That endpoint supports pagination and block range filtering +/// When a storage map contains many entries (> 1000), returning all entries in a single +/// RPC response creates performance issues. In such cases, `too_many_entries` is `true`, +/// `map_entries` is empty, and clients should use the `SyncStorageMaps` endpoint instead. #[derive(Debug, Clone, PartialEq, Eq)] pub struct AccountStorageMapDetails { pub slot_name: StorageSlotName, From eaf724240b7235cbf7bd572eed5c43be6fd44354 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 20:26:06 +0100 Subject: [PATCH 41/95] undo --- crates/proto/src/domain/account.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 0d7bfbde2d..5caaf03898 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -335,13 +335,14 @@ impl From for proto::account::AccountHeader { impl From for proto::account::AccountStorageHeader { fn from(value: AccountStorageHeader) -> Self { - let slots = Vec::from_iter(value.slots().map(|slot_header| { - proto::account::account_storage_header::StorageSlot { + let slots = value + .slots() + .map(|slot_header| proto::account::account_storage_header::StorageSlot { slot_name: slot_header.name().to_string(), slot_type: storage_slot_type_to_raw(slot_header.slot_type()), commitment: Some(proto::primitives::Digest::from(slot_header.value())), - } - })); + }) + .collect(); Self { slots } } From 72126e1f0f7b6a599bf433487ac59fb15239e287 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 20:32:09 +0100 Subject: [PATCH 42/95] one more enum --- crates/proto/src/domain/account.rs | 131 ++++++++++++++----------- crates/store/src/server/ntx_builder.rs | 3 +- 2 files changed, 75 insertions(+), 59 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 5caaf03898..863a247132 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -202,32 +202,36 @@ impl TryFrom let slot_name = StorageSlotName::new(slot_name)?; - // Extract map_entries from the MapEntries message - let map_entries = if let Some(entries) = entries { - entries - .entries - .into_iter() - .map(|entry| { - let key = entry - .key - .ok_or(proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry::missing_field( - stringify!(key), - ))? - .try_into()?; - let value = entry - .value - .ok_or(proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry::missing_field( - stringify!(value), - ))? - .try_into()?; - Ok((key, value)) - }) - .collect::, ConversionError>>()? + let entries = if too_many_entries { + StorageMapEntries::LimitExceeded } else { - Vec::new() + let map_entries = if let Some(entries) = entries { + entries + .entries +.into_iter() +.map(|entry| { +let key = entry +.key + .ok_or(proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry::missing_field( + stringify!(key), + ))? + .try_into()?; +let value = entry +.value + .ok_or(proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry::missing_field( + stringify!(value), + ))? + .try_into()?; + Ok((key, value)) + }) + .collect::, ConversionError>>()? + } else { + Vec::new() + }; + StorageMapEntries::Entries(map_entries) }; - Ok(Self { slot_name, too_many_entries, map_entries }) + Ok(Self { slot_name, entries }) } } @@ -438,24 +442,31 @@ impl From for proto::rpc::AccountVaultDetails { } } -/// Details about an account storage map slot, including overflow handling. +/// Storage map entries for an account storage slot. /// /// When a storage map contains many entries (> 1000), returning all entries in a single -/// RPC response creates performance issues. In such cases, `too_many_entries` is `true`, -/// `map_entries` is empty, and clients should use the `SyncStorageMaps` endpoint instead. +/// RPC response creates performance issues. In such cases, the `LimitExceeded` variant +/// indicates to the client to use the `SyncStorageMaps` endpoint instead. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct AccountStorageMapDetails { - pub slot_name: StorageSlotName, - pub too_many_entries: bool, +pub enum StorageMapEntries { + /// The map has too many entries to return inline. + /// Clients must use `SyncStorageMaps` endpoint instead. + LimitExceeded, - /// The storage map entries (key-value pairs). Empty if `too_many_entries` is `true`. + /// The storage map entries (key-value pairs), up to `MAX_RETURN_ENTRIES`. /// TODO: For partial responses, also include Merkle proofs and inner SMT nodes. - pub map_entries: Vec<(Word, Word)>, + Entries(Vec<(Word, Word)>), +} + +/// Details about an account storage map slot. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccountStorageMapDetails { + pub slot_name: StorageSlotName, + pub entries: StorageMapEntries, } impl AccountStorageMapDetails { /// Maximum number of storage map entries that can be returned in a single response. - /// Maps with more entries will have `too_many_entries = true` and empty `map_entries`. pub const MAX_RETURN_ENTRIES: usize = 1000; pub fn new(slot_name: StorageSlotName, slot_data: SlotData, storage_map: &StorageMap) -> Self { @@ -467,13 +478,15 @@ impl AccountStorageMapDetails { fn from_all_entries(slot_name: StorageSlotName, storage_map: &StorageMap) -> Self { if storage_map.num_entries() > Self::MAX_RETURN_ENTRIES { - Self::too_many_entries(slot_name) + Self { + slot_name, + entries: StorageMapEntries::LimitExceeded, + } } else { let map_entries = Vec::from_iter(storage_map.entries().map(|(k, v)| (*k, *v))); Self { slot_name, - too_many_entries: false, - map_entries, + entries: StorageMapEntries::Entries(map_entries), } } } @@ -484,20 +497,15 @@ impl AccountStorageMapDetails { storage_map: &StorageMap, ) -> Self { if keys.len() > Self::MAX_RETURN_ENTRIES { - Self::too_many_entries(slot_name) + Self { + slot_name, + entries: StorageMapEntries::LimitExceeded, + } } else { // TODO For now, we return all entries instead of specific keys with proofs Self::from_all_entries(slot_name, storage_map) } } - - pub fn too_many_entries(slot_name: StorageSlotName) -> Self { - Self { - slot_name, - too_many_entries: true, - map_entries: Vec::new(), - } - } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -665,21 +673,30 @@ impl From fn from(value: AccountStorageMapDetails) -> Self { use proto::rpc::account_storage_details::account_storage_map_details; - let AccountStorageMapDetails { slot_name, too_many_entries, map_entries } = value; + let AccountStorageMapDetails { slot_name, entries } = value; - let entries = Some(account_storage_map_details::MapEntries { - entries: Vec::from_iter(map_entries.into_iter().map(|(key, value)| { - account_storage_map_details::map_entries::StorageMapEntry { - key: Some(key.into()), - value: Some(value.into()), + match entries { + StorageMapEntries::LimitExceeded => Self { + slot_name: slot_name.to_string(), + too_many_entries: true, + entries: Some(account_storage_map_details::MapEntries { entries: Vec::new() }), + }, + StorageMapEntries::Entries(map_entries) => { + let entries = Some(account_storage_map_details::MapEntries { + entries: Vec::from_iter(map_entries.into_iter().map(|(key, value)| { + account_storage_map_details::map_entries::StorageMapEntry { + key: Some(key.into()), + value: Some(value.into()), + } + })), + }); + + Self { + slot_name: slot_name.to_string(), + too_many_entries: false, + entries, } - })), - }); - - Self { - slot_name: slot_name.to_string(), - too_many_entries, - entries, + }, } } } diff --git a/crates/store/src/server/ntx_builder.rs b/crates/store/src/server/ntx_builder.rs index 54422dc64e..4faa0d24d1 100644 --- a/crates/store/src/server/ntx_builder.rs +++ b/crates/store/src/server/ntx_builder.rs @@ -165,8 +165,7 @@ impl ntx_builder_server::NtxBuilder for StoreApi { ) -> Result, Status> { let account_ids = self.state.get_all_network_accounts().await.map_err(internal_error)?; - let account_ids = - Vec::from_iter(account_ids.into_iter().map(Into::into)); + let account_ids = Vec::from_iter(account_ids.into_iter().map(Into::into)); Ok(Response::new(proto::store::NetworkAccountIdList { account_ids })) } From be9071b9c983e2dfc019b01ae062c799b884c5c6 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 20:44:43 +0100 Subject: [PATCH 43/95] docs --- crates/store/src/db/models/queries/accounts.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 2c931f27f5..7457591329 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -699,8 +699,8 @@ pub(crate) fn insert_account_storage_map_value( /// /// This function queries the database tables to reconstruct a complete Account object: /// - Code from `account_codes` table -/// - Nonce from `accounts` table -/// - Storage from `account_storage_headers` and `account_storage_map_values` tables +/// - Nonce and storage header from `accounts` table +/// - Storage map entries from `account_storage_map_values` table /// - Vault from `account_vault_assets` table /// /// # Note From a0f8fc9c2376e314ca8f58c22fdd7d8e07328f3e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 21:01:03 +0100 Subject: [PATCH 44/95] unneces --- crates/store/src/server/rpc_api.rs | 111 +++++++++++++++-------------- 1 file changed, 59 insertions(+), 52 deletions(-) diff --git a/crates/store/src/server/rpc_api.rs b/crates/store/src/server/rpc_api.rs index de1da89c01..748cd07703 100644 --- a/crates/store/src/server/rpc_api.rs +++ b/crates/store/src/server/rpc_api.rs @@ -91,7 +91,7 @@ impl rpc_server::Rpc for StoreApi { let proofs = self.state.check_nullifiers(&nullifiers).await; Ok(Response::new(proto::rpc::CheckNullifiersResponse { - proofs: Vec::from_iter(convert(proofs)), + proofs: convert(proofs).collect(), })) } @@ -128,12 +128,13 @@ impl rpc_server::Rpc for StoreApi { .await .map_err(SyncNullifiersError::from)?; - let nullifiers = Vec::from_iter(nullifiers.into_iter().map(|nullifier_info| { - proto::rpc::sync_nullifiers_response::NullifierUpdate { + let nullifiers = nullifiers + .into_iter() + .map(|nullifier_info| proto::rpc::sync_nullifiers_response::NullifierUpdate { nullifier: Some(nullifier_info.nullifier.into()), block_num: nullifier_info.block_num.as_u32(), - } - })); + }) + .collect(); Ok(Response::new(proto::rpc::SyncNullifiersResponse { pagination_info: Some(proto::rpc::PaginationInfo { @@ -169,24 +170,27 @@ impl rpc_server::Rpc for StoreApi { .await .map_err(internal_error)?; - let accounts = Vec::from_iter(state.account_updates.into_iter().map(|account_info| { - proto::account::AccountSummary { + let accounts = state + .account_updates + .into_iter() + .map(|account_info| proto::account::AccountSummary { account_id: Some(account_info.account_id.into()), account_commitment: Some(account_info.account_commitment.into()), block_num: account_info.block_num.as_u32(), - } - })); - - let transactions = - Vec::from_iter(state.transactions.into_iter().map(|transaction_summary| { - proto::transaction::TransactionSummary { - account_id: Some(transaction_summary.account_id.into()), - block_num: transaction_summary.block_num.as_u32(), - transaction_id: Some(transaction_summary.transaction_id.into()), - } - })); + }) + .collect(); + + let transactions = state + .transactions + .into_iter() + .map(|transaction_summary| proto::transaction::TransactionSummary { + account_id: Some(transaction_summary.account_id.into()), + block_num: transaction_summary.block_num.as_u32(), + transaction_id: Some(transaction_summary.transaction_id.into()), + }) + .collect(); - let notes = Vec::from_iter(state.notes.into_iter().map(Into::into)); + let notes = state.notes.into_iter().map(Into::into).collect(); Ok(Response::new(proto::rpc::SyncStateResponse { chain_tip: self.state.latest_block_num().await.as_u32(), @@ -225,7 +229,7 @@ impl rpc_server::Rpc for StoreApi { let (state, mmr_proof, last_block_included) = self.state.sync_notes(request.note_tags, block_range).await?; - let notes = Vec::from_iter(state.notes.into_iter().map(Into::into)); + let notes = state.notes.into_iter().map(Into::into).collect(); Ok(Response::new(proto::rpc::SyncNotesResponse { pagination_info: Some(proto::rpc::PaginationInfo { @@ -264,16 +268,16 @@ impl rpc_server::Rpc for StoreApi { let note_ids: Vec = convert_digests_to_words::(note_ids)?; - let note_ids = Vec::from_iter(note_ids.into_iter().map(NoteId::from_raw)); + let note_ids: Vec = note_ids.into_iter().map(NoteId::from_raw).collect(); - let notes = Vec::from_iter( - self.state - .get_notes_by_id(note_ids) - .await - .map_err(GetNotesByIdError::from)? - .into_iter() - .map(Into::into), - ); + let notes = self + .state + .get_notes_by_id(note_ids) + .await + .map_err(GetNotesByIdError::from)? + .into_iter() + .map(Into::into) + .collect(); Ok(Response::new(proto::note::CommittedNoteList { notes })) } @@ -383,14 +387,17 @@ impl rpc_server::Rpc for StoreApi { .await .map_err(SyncAccountVaultError::from)?; - let updates = Vec::from_iter(updates.into_iter().map(|update| { - let vault_key: Word = update.vault_key.into(); - proto::rpc::AccountVaultUpdate { - vault_key: Some(vault_key.into()), - asset: update.asset.map(Into::into), - block_num: update.block_num.as_u32(), - } - })); + let updates = updates + .into_iter() + .map(|update| { + let vault_key: Word = update.vault_key.into(); + proto::rpc::AccountVaultUpdate { + vault_key: Some(vault_key.into()), + asset: update.asset.map(Into::into), + block_num: update.block_num.as_u32(), + } + }) + .collect(); Ok(Response::new(proto::rpc::SyncAccountVaultResponse { pagination_info: Some(proto::rpc::PaginationInfo { @@ -438,14 +445,16 @@ impl rpc_server::Rpc for StoreApi { .await .map_err(SyncStorageMapsError::from)?; - let updates = Vec::from_iter(storage_maps_page.values.into_iter().map(|map_value| { - proto::rpc::StorageMapUpdate { + let updates = storage_maps_page + .values + .into_iter() + .map(|map_value| proto::rpc::StorageMapUpdate { slot_name: map_value.slot_name.to_string(), key: Some(map_value.key.into()), value: Some(map_value.value.into()), block_num: map_value.block_num.as_u32(), - } - })); + }) + .collect(); Ok(Response::new(proto::rpc::SyncStorageMapsResponse { pagination_info: Some(proto::rpc::PaginationInfo { @@ -553,23 +562,21 @@ impl rpc_server::Rpc for StoreApi { .map_err(SyncTransactionsError::from)?; // Create a map from note ID to note record for efficient lookup - let note_map: std::collections::HashMap<_, _> = std::collections::HashMap::from_iter( - all_note_records - .into_iter() - .map(|note_record| (note_record.note_id, note_record)), - ); + let note_map: std::collections::HashMap<_, _> = all_note_records + .into_iter() + .map(|note_record| (note_record.note_id, note_record)) + .collect(); // Convert database TransactionRecord to proto TransactionRecord let mut transactions = Vec::with_capacity(transaction_records_db.len()); for tx_header in transaction_records_db { // Get note records for this transaction's output notes - let note_records = Vec::from_iter( - tx_header - .output_notes - .iter() - .filter_map(|note_id| note_map.get(¬e_id.as_word()).cloned()), - ); + let note_records: Vec<_> = tx_header + .output_notes + .iter() + .filter_map(|note_id| note_map.get(¬e_id.as_word()).cloned()) + .collect(); // Convert to proto using the helper method let proto_record = tx_header.into_proto_with_note_records(note_records); From 88c058b9e634da5639228316f6e8d62436f2ed8b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 21:12:46 +0100 Subject: [PATCH 45/95] simplify --- crates/store/src/state.rs | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index d803c2767e..82cb69b762 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -508,8 +508,13 @@ impl State { // Step 2: Extract map slots and their entries using InnerForest helper let map_slots_to_populate = InnerForest::extract_map_slots_from_storage(&account_storages); - // Step 3: Update the forest with new SMTs - self.populate_forest_with_storage_maps(map_slots_to_populate, block_num).await?; + if map_slots_to_populate.is_empty() { + return Ok(()); + } + + // Step 3: Acquire write lock and update the forest with new SMTs + let mut forest_guard = self.forest.write().await; + forest_guard.populate_storage_maps(map_slots_to_populate, block_num); Ok(()) } @@ -531,27 +536,6 @@ impl State { Ok(account_storages) } - /// Populates the forest with storage map SMTs for the given slots - #[instrument(target = COMPONENT, skip_all, fields(num_slots = map_slots.len()))] - #[allow(clippy::type_complexity)] - async fn populate_forest_with_storage_maps( - &self, - map_slots: Vec<(AccountId, u8, Vec<(&Word, &Word)>)>, - block_num: BlockNumber, - ) -> Result<(), ApplyBlockError> { - if map_slots.is_empty() { - return Ok(()); - } - - // Acquire write lock once for all updates - let mut forest_guard = self.forest.write().await; - - // Delegate to InnerForest for the actual population logic - forest_guard.populate_storage_maps(map_slots, block_num); - - Ok(()) - } - /// Updates vault SMTs in the forest for changed accounts #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = changed_account_ids.len()))] async fn update_vaults_in_forest( From b84f25fe54b7c701da69d862b7ccad3ac4b8d67b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 21:25:34 +0100 Subject: [PATCH 46/95] misleading --- crates/proto/src/domain/account.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 863a247132..7aa49287d0 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -90,7 +90,6 @@ impl From<&AccountSummary> for proto::account::AccountSummary { } } -// TODO #[deprecated(note = "avoid this type, details will be `None` always!")] #[derive(Debug, PartialEq)] pub struct AccountInfo { pub summary: AccountSummary, From 25b5550b115f150fcba19667cad044a3a69ed84b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 21:30:07 +0100 Subject: [PATCH 47/95] bound --- crates/proto/src/domain/account.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 7aa49287d0..e3e29fbab7 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -353,9 +353,9 @@ impl From for proto::account::AccountStorageHeader { /// Account vault details /// -/// When an account contains a large number of assets (> 1000), including all assets -/// in a single RPC response creates performance issues. In such cases, the `LimitExceeded` -/// variant indicates to the client to use the `SyncAccountVault` endpoint instead. +/// When an account contains a large number of assets (> [`AccountVaultDetails::MAX_RETURN_ENTRIES`]), +/// including all assets in a single RPC response creates performance issues. In such cases, +/// the `LimitExceeded` variant indicates to the client to use the `SyncAccountVault` endpoint instead. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AccountVaultDetails { /// The vault has too many assets to return inline. @@ -369,7 +369,7 @@ pub enum AccountVaultDetails { impl AccountVaultDetails { /// Maximum number of vault entries that can be returned in a single response. /// Accounts with more assets will have `LimitExceeded` variant. - const MAX_RETURN_ENTRIES: usize = 1000; + pub const MAX_RETURN_ENTRIES: usize = 1000; pub fn new(vault: &AssetVault) -> Self { if vault.assets().nth(Self::MAX_RETURN_ENTRIES).is_some() { @@ -443,9 +443,9 @@ impl From for proto::rpc::AccountVaultDetails { /// Storage map entries for an account storage slot. /// -/// When a storage map contains many entries (> 1000), returning all entries in a single -/// RPC response creates performance issues. In such cases, the `LimitExceeded` variant -/// indicates to the client to use the `SyncStorageMaps` endpoint instead. +/// When a storage map contains many entries (> [`AccountStorageMapDetails::MAX_RETURN_ENTRIES`]), +/// returning all entries in a single RPC response creates performance issues. In such cases, +/// the `LimitExceeded` variant indicates to the client to use the `SyncStorageMaps` endpoint instead. #[derive(Debug, Clone, PartialEq, Eq)] pub enum StorageMapEntries { /// The map has too many entries to return inline. From 31dacddfe5870a0cdc825dd41276e4f44d0bbb94 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 21:52:07 +0100 Subject: [PATCH 48/95] fmt --- crates/proto/src/domain/account.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index e3e29fbab7..03eebbdf1f 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -353,9 +353,10 @@ impl From for proto::account::AccountStorageHeader { /// Account vault details /// -/// When an account contains a large number of assets (> [`AccountVaultDetails::MAX_RETURN_ENTRIES`]), -/// including all assets in a single RPC response creates performance issues. In such cases, -/// the `LimitExceeded` variant indicates to the client to use the `SyncAccountVault` endpoint instead. +/// When an account contains a large number of assets (> +/// [`AccountVaultDetails::MAX_RETURN_ENTRIES`]), including all assets in a single RPC response +/// creates performance issues. In such cases, the `LimitExceeded` variant indicates to the client +/// to use the `SyncAccountVault` endpoint instead. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AccountVaultDetails { /// The vault has too many assets to return inline. @@ -445,7 +446,8 @@ impl From for proto::rpc::AccountVaultDetails { /// /// When a storage map contains many entries (> [`AccountStorageMapDetails::MAX_RETURN_ENTRIES`]), /// returning all entries in a single RPC response creates performance issues. In such cases, -/// the `LimitExceeded` variant indicates to the client to use the `SyncStorageMaps` endpoint instead. +/// the `LimitExceeded` variant indicates to the client to use the `SyncStorageMaps` endpoint +/// instead. #[derive(Debug, Clone, PartialEq, Eq)] pub enum StorageMapEntries { /// The map has too many entries to return inline. From 55f4a46aea0a255e544082ce1cce3c85fccf29ae Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 18 Dec 2025 21:53:57 +0100 Subject: [PATCH 49/95] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85f7f989ef..0f510a100f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - [BREAKING] Added block signing capabilities to Validator component and updated gensis bootstrap to sign blocks with configured signer ([#1426](https://github.com/0xMiden/miden-node/pull/1426)). - Reduced default block interval from `5s` to `2s` ([#1438](https://github.com/0xMiden/miden-node/pull/1438)). - Increased retained account tree history from 33 to 100 blocks to account for the reduced block interval ([#1438](https://github.com/0xMiden/miden-node/pull/1438)). +- Refactor account table and introduce tracking forest ([#1394](https://github.com/0xMiden/miden-node/pull/1394)). ### Fixes From bf67ce87ad4cc7657fcd1a1fdb133fac61585a41 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 19 Dec 2025 14:10:32 +0100 Subject: [PATCH 50/95] 0 ->1; 1->0 --- crates/store/src/db/models/conv.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/store/src/db/models/conv.rs b/crates/store/src/db/models/conv.rs index 8f148a6177..3600f56d9a 100644 --- a/crates/store/src/db/models/conv.rs +++ b/crates/store/src/db/models/conv.rs @@ -141,8 +141,8 @@ impl SqlTypeConvert for StorageSlotType { struct ValueError(i32); Ok(match raw { - 0 => StorageSlotType::Map, - 1 => StorageSlotType::Value, + 0 => StorageSlotType::Value, + 1 => StorageSlotType::Map, invalid => { return Err(Self::map_err(ValueError(invalid))); }, @@ -152,8 +152,8 @@ impl SqlTypeConvert for StorageSlotType { #[inline(always)] fn to_raw_sql(self) -> Self::Raw { match self { - StorageSlotType::Map => 0, - StorageSlotType::Value => 1, + StorageSlotType::Value => 0, + StorageSlotType::Map => 1, } } } From 0c0e32b76ce9d215df457d8efa9093b22d62395a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 19 Dec 2025 14:19:45 +0100 Subject: [PATCH 51/95] avoid full paths --- crates/store/src/db/mod.rs | 6 +++--- crates/store/src/errors.rs | 6 +++--- crates/store/src/state.rs | 21 +++++++++++---------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 78c366ba12..704f70a6e4 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -8,7 +8,7 @@ use miden_lib::utils::{Deserializable, Serializable}; use miden_node_proto::domain::account::{AccountInfo, AccountSummary, NetworkAccountPrefix}; use miden_node_proto::generated as proto; use miden_objects::Word; -use miden_objects::account::AccountId; +use miden_objects::account::{AccountHeader, AccountId, AccountStorage}; use miden_objects::asset::{Asset, AssetVaultKey}; use miden_objects::block::{BlockHeader, BlockNoteIndex, BlockNumber, ProvenBlock}; use miden_objects::crypto::merkle::SparseMerklePath; @@ -436,7 +436,7 @@ impl Db { &self, account_id: AccountId, block_num: BlockNumber, - ) -> Result { + ) -> Result { self.transact("Get account storage at block", move |conn| { queries::select_account_storage_at_block(conn, account_id, block_num) }) @@ -491,7 +491,7 @@ impl Db { &self, account_id: AccountId, block_num: BlockNumber, - ) -> Result> { + ) -> Result> { self.transact("Get account header at block", move |conn| { queries::select_account_header_at_block(conn, account_id, block_num) }) diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index f809378fbf..6e67954b89 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -10,7 +10,7 @@ use miden_objects::account::AccountId; use miden_objects::block::BlockNumber; use miden_objects::crypto::merkle::MmrError; use miden_objects::crypto::utils::DeserializationError; -use miden_objects::note::Nullifier; +use miden_objects::note::{NoteId, Nullifier}; use miden_objects::transaction::OutputNote; use miden_objects::{ AccountDeltaError, @@ -453,9 +453,9 @@ pub enum GetNotesByIdError { #[error("malformed note ID")] DeserializationFailed(#[from] ConversionError), #[error("note {0} not found")] - NoteNotFound(miden_objects::note::NoteId), + NoteNotFound(NoteId), #[error("note {0} is not public")] - NoteNotPublic(miden_objects::note::NoteId), + NoteNotPublic(NoteId), } // GET NOTE SCRIPT BY ROOT ERRORS diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 82cb69b762..572d1870c0 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -23,13 +23,14 @@ use miden_node_proto::domain::account::{ use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; -use miden_objects::account::{AccountId, StorageSlotContent}; +use miden_objects::account::{AccountId, AccountStorage, StorageSlotContent}; use miden_objects::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_objects::block::nullifier_tree::{NullifierTree, NullifierWitness}; -use miden_objects::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, ProvenBlock}; +use miden_objects::block::{BlockAccountUpdate, BlockHeader, BlockInputs, BlockNoteTree, BlockNumber, Blockchain, ProvenBlock}; use miden_objects::crypto::merkle::{ Forest, LargeSmt, + LargeSmtError, MemoryStorage, Mmr, MmrDelta, @@ -148,7 +149,7 @@ impl State { let block_headers = db.select_all_block_headers().await?; let latest_block_num = block_headers .last() - .map_or(BlockNumber::GENESIS, miden_objects::block::BlockHeader::block_num); + .map_or(BlockNumber::GENESIS, BlockHeader::block_num); let account_tree = load_account_tree(&mut db, latest_block_num).await?; let nullifier_tree = load_nullifier_tree(&mut db).await?; @@ -344,7 +345,7 @@ impl State { .map(|(note_index, note)| (note_index, note.id(), *note.metadata())), ); let note_tree = - miden_objects::block::BlockNoteTree::with_entries(note_tree_entries.iter().copied()) + BlockNoteTree::with_entries(note_tree_entries.iter().copied()) .map_err(|e| InvalidBlockError::FailedToBuildNoteTree(e.to_string()))?; if note_tree.root() != header.note_root() { return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); @@ -394,7 +395,7 @@ impl State { .body() .updated_accounts() .iter() - .map(miden_objects::block::BlockAccountUpdate::account_id), + .map(BlockAccountUpdate::account_id), ); // The DB and in-memory state updates need to be synchronized and are partially @@ -525,7 +526,7 @@ impl State { &self, account_ids: &[AccountId], block_num: BlockNumber, - ) -> Result, ApplyBlockError> { + ) -> Result, ApplyBlockError> { let mut account_storages = Vec::with_capacity(account_ids.len()); for &account_id in account_ids { @@ -1061,7 +1062,7 @@ impl State { &self, account_id: AccountId, block_num: BlockNumber, - ) -> Result { + ) -> Result { // Validate block exists in the blockchain before querying the database self.validate_block_exists(block_num).await?; @@ -1072,7 +1073,7 @@ impl State { pub async fn get_latest_account_storage( &self, account_id: AccountId, - ) -> Result { + ) -> Result { self.db.select_latest_account_storage(account_id).await } @@ -1369,10 +1370,10 @@ async fn load_account_tree( let smt = LargeSmt::with_entries(MemoryStorage::default(), smt_entries).map_err(|e| match e { - miden_objects::crypto::merkle::LargeSmtError::Merkle(merkle_error) => { + LargeSmtError::Merkle(merkle_error) => { StateInitializationError::DatabaseError(DatabaseError::MerkleError(merkle_error)) }, - miden_objects::crypto::merkle::LargeSmtError::Storage(err) => { + LargeSmtError::Storage(err) => { // large_smt::StorageError is not `Sync` and hence `context` cannot be called // which we want to and do StateInitializationError::AccountTreeIoError(err.as_report()) From ec4318ebcacda4a2fda1ea7782c3eb3afad6aaac Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 19 Dec 2025 17:35:49 +0100 Subject: [PATCH 52/95] fn --- crates/proto/src/domain/account.rs | 8 +- crates/store/src/inner_forest.rs | 437 ++++++++++++++++++++++++----- crates/store/src/state.rs | 108 +++++-- 3 files changed, 460 insertions(+), 93 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 03eebbdf1f..60c840154b 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -544,16 +544,16 @@ impl From for proto::rpc::AccountStorageDetails { const fn storage_slot_type_from_raw(slot_type: u32) -> Result { Ok(match slot_type { - 0 => StorageSlotType::Map, - 1 => StorageSlotType::Value, + 0 => StorageSlotType::Value, + 1 => StorageSlotType::Map, _ => return Err(ConversionError::EnumDiscriminantOutOfRange), }) } const fn storage_slot_type_to_raw(slot_type: StorageSlotType) -> u32 { match slot_type { - StorageSlotType::Map => 0, - StorageSlotType::Value => 1, + StorageSlotType::Value => 0, + StorageSlotType::Map => 1, } } diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index d817551478..8dda1d25a9 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -1,19 +1,26 @@ use std::collections::BTreeMap; -use miden_objects::account::{AccountId, AccountStorage, StorageSlotContent}; +use miden_objects::account::delta::{AccountStorageDelta, AccountVaultDelta}; +use miden_objects::account::{AccountId, NonFungibleDeltaAction, StorageSlotName}; +use miden_objects::asset::{Asset, FungibleAsset}; use miden_objects::block::BlockNumber; -use miden_objects::crypto::merkle::SmtForest; +use miden_objects::crypto::merkle::{EmptySubtreeRoots, SMT_DEPTH, SmtForest}; use miden_objects::{EMPTY_WORD, Word}; +// Type aliases to reduce complexity +type MapSlotEntries = Vec<(Word, Word)>; +type StorageMapSlot = (AccountId, StorageSlotName, MapSlotEntries); +type VaultEntries = Vec<(Word, Word)>; + /// Container for forest-related state that needs to be updated atomically. pub(crate) struct InnerForest { /// `SmtForest` for efficient account storage reconstruction. /// Populated during block import with storage and vault SMTs. pub(crate) storage_forest: SmtForest, - /// Maps (`account_id`, `slot_index`, `block_num`) to SMT root. + /// Maps (`account_id`, `slot_name`, `block_num`) to SMT root. /// Populated during block import for all storage map slots. - storage_roots: BTreeMap<(AccountId, u8, BlockNumber), Word>, + storage_roots: BTreeMap<(AccountId, StorageSlotName, BlockNumber), Word>, /// Maps (`account_id`, `block_num`) to vault SMT root. /// Tracks asset vault versions across all blocks with structural sharing. @@ -29,117 +36,409 @@ impl InnerForest { } } - /// Extracts map-type storage slots and their entries from account storage data. + /// Returns the root of an empty SMT. + fn empty_smt_root() -> Word { + *EmptySubtreeRoots::entry(SMT_DEPTH, 0) + } + + /// Updates the forest with account vault and storage changes from a delta. /// - /// This is a helper method to prepare data for populating the forest with storage maps. - /// It iterates through all accounts' storage slots and collects only the map-type slots - /// with their entries. + /// This is the unified interface for updating all account state in the forest. + /// It processes both vault and storage map deltas and updates the forest accordingly. /// /// # Arguments /// - /// * `account_storages` - Slice of `(account_id, storage)` tuples from database + /// * `block_num` - Block number for which these changes are being applied + /// * `account_id` - The account being updated + /// * `vault_delta` - Changes to the account's asset vault + /// * `storage_delta` - Changes to the account's storage maps + pub(crate) fn update_account( + &mut self, + block_num: BlockNumber, + account_id: AccountId, + vault_delta: &AccountVaultDelta, + storage_delta: &AccountStorageDelta, + ) { + // Update vault if there are any changes + if !vault_delta.is_empty() { + self.update_account_vault(block_num, account_id, vault_delta); + } + + // Update storage maps if there are any changes + if !storage_delta.is_empty() { + self.update_account_storage(block_num, account_id, storage_delta); + } + } + + /// Updates the forest with vault changes from a delta. /// - /// # Returns + /// Processes both fungible and non-fungible asset changes, building entries + /// for the vault SMT and tracking the new root. /// - /// Vec of `(account_id, slot_index, entries)` tuples ready for forest population - #[allow(clippy::type_complexity)] - pub(crate) fn extract_map_slots_from_storage( - account_storages: &[(AccountId, AccountStorage)], - ) -> Vec<(AccountId, u8, Vec<(&Word, &Word)>)> { - let mut map_slots = Vec::new(); + /// # Arguments + /// + /// * `block_num` - Block number for this update + /// * `account_id` - The account being updated + /// * `vault_delta` - Changes to the account's asset vault + fn update_account_vault( + &mut self, + block_num: BlockNumber, + account_id: AccountId, + vault_delta: &AccountVaultDelta, + ) { + let prev_block_num = block_num.parent().unwrap_or_default(); + let prev_root = self + .vault_roots + .get(&(account_id, prev_block_num)) + .copied() + .unwrap_or_else(Self::empty_smt_root); - for (account_id, storage) in account_storages { - for (slot_idx, slot) in storage.slots().iter().enumerate() { - if let StorageSlotContent::Map(storage_map) = slot.content() { - let entries = Vec::from_iter(storage_map.entries()); - map_slots.push((*account_id, slot_idx as u8, entries)); - } + // Collect all vault entry updates + let mut entries = Vec::new(); + + // Process fungible assets - these require special handling to get current amounts + // Note: We rely on the delta containing the updated amounts, not just the changes + for (faucet_id, amount) in vault_delta.fungible().iter() { + let amount_u64 = (*amount).try_into().expect("Amount should be non-negative"); + let asset: Asset = FungibleAsset::new(*faucet_id, amount_u64) + .expect("Valid fungible asset from delta") + .into(); + entries.push((asset.vault_key().into(), Word::from(asset))); + } + + // Process non-fungible assets + for (asset, action) in vault_delta.non_fungible().iter() { + match action { + NonFungibleDeltaAction::Add => { + entries + .push((asset.vault_key().into(), Word::from(Asset::NonFungible(*asset)))); + }, + NonFungibleDeltaAction::Remove => { + entries.push((asset.vault_key().into(), EMPTY_WORD)); + }, } } - tracing::debug!(target: crate::COMPONENT, num_map_slots = map_slots.len()); - map_slots + if !entries.is_empty() { + let updated_root = self + .storage_forest + .batch_insert(prev_root, entries.iter().copied()) + .expect("Forest insertion should succeed"); + + self.vault_roots.insert((account_id, block_num), updated_root); + + tracing::debug!( + target: crate::COMPONENT, + account_id = %account_id, + block_num = %block_num, + vault_entries = entries.len(), + "Updated vault in forest" + ); + } } - /// Populates the forest with storage map SMTs for the given slots. + /// Updates the forest with storage map changes from a delta. /// - /// This method builds SMTs from the provided entries and tracks their roots, - /// enabling efficient historical queries with structural sharing. + /// Processes storage map slot deltas, building SMTs for each modified slot + /// and tracking the new roots. /// /// # Arguments /// - /// * `map_slots` - Vec of `(account_id, slot_index, entries)` tuples - /// * `block_num` - Block number for which these SMTs are being created - #[allow(clippy::type_complexity)] - pub(crate) fn populate_storage_maps( + /// * `block_num` - Block number for this update + /// * `account_id` - The account being updated + /// * `storage_delta` - Changes to the account's storage maps + fn update_account_storage( &mut self, - map_slots: Vec<(AccountId, u8, Vec<(&Word, &Word)>)>, block_num: BlockNumber, + account_id: AccountId, + storage_delta: &AccountStorageDelta, ) { let prev_block_num = block_num.parent().unwrap_or_default(); - for (account_id, slot_idx, entries) in map_slots { - // Get previous root for structural sharing + for (slot_name, map_delta) in storage_delta.maps() { let prev_root = self .storage_roots - .get(&(account_id, slot_idx, prev_block_num)) + .get(&(account_id, slot_name.clone(), prev_block_num)) .copied() - .unwrap_or(EMPTY_WORD); + .unwrap_or_else(Self::empty_smt_root); + + // Collect entries from the delta + let entries = map_delta + .entries() + .iter() + .map(|(key, value)| ((*key).into(), *value)) + .collect::>(); + + if !entries.is_empty() { + let updated_root = self + .storage_forest + .batch_insert(prev_root, entries.iter().copied()) + .expect("Forest insertion should succeed"); + + self.storage_roots + .insert((account_id, slot_name.clone(), block_num), updated_root); + + tracing::debug!( + target: crate::COMPONENT, + account_id = %account_id, + block_num = %block_num, + slot_name = ?slot_name, + entries = entries.len(), + "Updated storage map in forest" + ); + } + } + } + + // LEGACY DB-BASED POPULATION METHODS + // ================================================================================================ + // These methods are used during initial State::load() where deltas are not available. + // They populate the forest from full database state rather than incremental deltas. + // + // For block application, prefer `update_account()` which uses deltas directly. + + /// Populates storage map SMTs in the forest from full database state. + /// + /// **DEPRECATED for block application**: Use `update_account()` with deltas instead. + /// This method is primarily used during `State::load()` where deltas are not available. + /// + /// # Arguments + /// + /// * `map_slots_to_populate` - List of (`account_id`, `slot_name`, entries) tuples + /// * `block_num` - Block number for which this state applies + #[allow(dead_code)] // Used only during State::load + pub(crate) fn populate_storage_maps( + &mut self, + map_slots_to_populate: Vec, + block_num: BlockNumber, + ) { + for (account_id, slot_name, entries) in map_slots_to_populate { + if entries.is_empty() { + continue; + } - // Build new SMT from entries let updated_root = self .storage_forest - .batch_insert(prev_root, entries.into_iter().map(|(k, v)| (*k, *v))) - .expect("Forest insertion should always succeed with valid entries"); + .batch_insert(Self::empty_smt_root(), entries.iter().copied()) + .expect("Forest insertion should succeed"); - // Track the new root - self.storage_roots.insert((account_id, slot_idx, block_num), updated_root); - } + self.storage_roots + .insert((account_id, slot_name.clone(), block_num), updated_root); - tracing::debug!( - target: crate::COMPONENT, - total_tracked_roots = self.storage_roots.len(), - "Updated storage map roots" - ); + tracing::debug!( + target: crate::COMPONENT, + account_id = %account_id, + block_num = %block_num, + slot_name = ?slot_name, + entries = entries.len(), + "Populated storage map in forest from DB" + ); + } } - /// Populates the forest with vault SMTs for the given accounts. + /// Populates vault SMTs in the forest from full database state. /// - /// This method builds vault SMTs from the provided asset entries and tracks their roots, - /// enabling efficient historical queries with structural sharing. + /// **DEPRECATED for block application**: Use `update_account()` with deltas instead. + /// This method is primarily used during `State::load()` where deltas are not available. /// /// # Arguments /// - /// * `vault_entries` - Vec of `(account_id, entries)` tuples where entries are (key, value) - /// pairs - /// * `block_num` - Block number for which these vault SMTs are being created + /// * `vault_entries_to_populate` - List of (`account_id`, `vault_entries`) tuples where entries + /// are (key, value) Word pairs + /// * `block_num` - Block number for which this state applies + #[allow(dead_code)] // Used only during State::load pub(crate) fn populate_vaults( &mut self, - vault_entries: Vec<(AccountId, Vec<(Word, Word)>)>, + vault_entries_to_populate: Vec<(AccountId, VaultEntries)>, block_num: BlockNumber, ) { - let prev_block_num = block_num.parent().unwrap_or_default(); - - for (account_id, entries) in vault_entries { - let prev_root = self - .vault_roots - .get(&(account_id, prev_block_num)) - .copied() - .unwrap_or(EMPTY_WORD); + for (account_id, entries) in vault_entries_to_populate { + if entries.is_empty() { + continue; + } let updated_root = self .storage_forest - .batch_insert(prev_root, entries) - .expect("Database is consistent and always allows constructing a smt or forest"); + .batch_insert(Self::empty_smt_root(), entries.iter().copied()) + .expect("Forest insertion should succeed"); - // Track the new vault root self.vault_roots.insert((account_id, block_num), updated_root); + + tracing::debug!( + target: crate::COMPONENT, + account_id = %account_id, + block_num = %block_num, + vault_entries = entries.len(), + "Populated vault in forest from DB" + ); } + } + + /// Helper method to extract storage map slots from `AccountStorage` objects. + /// + /// Used by the legacy DB-based population path during `State::load()`. + /// + /// # Returns + /// + /// Vector of (`account_id`, `slot_name`, entries) tuples ready for forest population + pub(crate) fn extract_map_slots_from_storage( + account_storages: &[(AccountId, miden_objects::account::AccountStorage)], + ) -> Vec { + use miden_objects::account::StorageSlotContent; + + let mut map_slots = Vec::new(); + + for (account_id, storage) in account_storages { + for slot in storage.slots() { + if let StorageSlotContent::Map(map) = slot.content() { + let entries: Vec<_> = map.entries().map(|(k, v)| (*k, *v)).collect(); + + if !entries.is_empty() { + map_slots.push((*account_id, slot.name().clone(), entries)); + } + } + } + } + + map_slots + } +} + +#[cfg(test)] +mod tests { + use super::*; + use miden_objects::asset::{Asset, FungibleAsset}; + use miden_objects::testing::account_id::{ + ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET, + ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE, + }; + + fn test_account() -> AccountId { + AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap() + } + + fn test_faucet() -> AccountId { + AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap() + } + + fn create_fungible_asset(faucet_id: AccountId, amount: u64) -> Asset { + FungibleAsset::new(faucet_id, amount).unwrap().into() + } + + #[test] + fn test_empty_smt_root_is_recognized() { + use miden_objects::crypto::merkle::Smt; - tracing::debug!( - target: crate::COMPONENT, - total_vault_roots = self.vault_roots.len(), - "Updated vault roots" + let empty_root = InnerForest::empty_smt_root(); + + // Verify an empty SMT has the expected root + assert_eq!(Smt::default().root(), empty_root); + + // Test that SmtForest accepts this root in batch_insert + let mut forest = SmtForest::new(); + let entries = vec![(Word::from([1u32, 2, 3, 4]), Word::from([5u32, 6, 7, 8]))]; + + assert!(forest.batch_insert(empty_root, entries).is_ok()); + } + + #[test] + fn test_inner_forest_basic_initialization() { + let forest = InnerForest::new(); + assert!(forest.storage_roots.is_empty()); + assert!(forest.vault_roots.is_empty()); + } + + #[test] + fn test_update_account_with_empty_deltas() { + let mut forest = InnerForest::new(); + let account_id = test_account(); + let block_num = BlockNumber::GENESIS.child(); + + let vault_delta = AccountVaultDelta::default(); + let storage_delta = AccountStorageDelta::default(); + + forest.update_account(block_num, account_id, &vault_delta, &storage_delta); + + // Empty deltas should not create entries + assert!(!forest.vault_roots.contains_key(&(account_id, block_num))); + assert!(forest.storage_roots.is_empty()); + } + + #[test] + fn test_update_vault_with_fungible_asset() { + let mut forest = InnerForest::new(); + let account_id = test_account(); + let faucet_id = test_faucet(); + let block_num = BlockNumber::GENESIS.child(); + + let asset = create_fungible_asset(faucet_id, 100); + let mut vault_delta = AccountVaultDelta::default(); + vault_delta.add_asset(asset).unwrap(); + + forest.update_account(block_num, account_id, &vault_delta, &AccountStorageDelta::default()); + + let vault_root = forest.vault_roots[&(account_id, block_num)]; + assert_ne!(vault_root, EMPTY_WORD); + } + + #[test] + fn test_compare_delta_vs_db_vault_with_fungible_asset() { + let account_id = test_account(); + let faucet_id = test_faucet(); + let block_num = BlockNumber::GENESIS.child(); + let asset = create_fungible_asset(faucet_id, 100); + + // Approach 1: Delta-based update + let mut forest_delta = InnerForest::new(); + let mut vault_delta = AccountVaultDelta::default(); + vault_delta.add_asset(asset).unwrap(); + forest_delta.update_account( + block_num, + account_id, + &vault_delta, + &AccountStorageDelta::default(), ); + + // Approach 2: DB-based population + let mut forest_db = InnerForest::new(); + let vault_entries = vec![(asset.vault_key().into(), Word::from(asset))]; + forest_db.populate_vaults(vec![(account_id, vault_entries)], block_num); + + // Both approaches must produce identical roots + let root_delta = forest_delta.vault_roots.get(&(account_id, block_num)).unwrap(); + let root_db = forest_db.vault_roots.get(&(account_id, block_num)).unwrap(); + + assert_eq!(root_delta, root_db); + assert_ne!(*root_delta, EMPTY_WORD); + } + + #[test] + fn test_slot_names_are_tracked() { + let forest = InnerForest::new(); + let _: &BTreeMap<(AccountId, StorageSlotName, BlockNumber), Word> = &forest.storage_roots; + } + + #[test] + fn test_incremental_vault_updates() { + let mut forest = InnerForest::new(); + let account_id = test_account(); + let faucet_id = test_faucet(); + let storage_delta = AccountStorageDelta::default(); + + // Block 1: 100 tokens + let block_1 = BlockNumber::GENESIS.child(); + let mut vault_delta_1 = AccountVaultDelta::default(); + vault_delta_1.add_asset(create_fungible_asset(faucet_id, 100)).unwrap(); + forest.update_account(block_1, account_id, &vault_delta_1, &storage_delta); + let root_1 = forest.vault_roots[&(account_id, block_1)]; + + // Block 2: 150 tokens + let block_2 = block_1.child(); + let mut vault_delta_2 = AccountVaultDelta::default(); + vault_delta_2.add_asset(create_fungible_asset(faucet_id, 150)).unwrap(); + forest.update_account(block_2, account_id, &vault_delta_2, &storage_delta); + let root_2 = forest.vault_roots[&(account_id, block_2)]; + + assert_ne!(root_1, root_2); } } diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 572d1870c0..048b9c8c89 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -23,10 +23,18 @@ use miden_node_proto::domain::account::{ use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; +use miden_objects::account::delta::AccountUpdateDetails; use miden_objects::account::{AccountId, AccountStorage, StorageSlotContent}; use miden_objects::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_objects::block::nullifier_tree::{NullifierTree, NullifierWitness}; -use miden_objects::block::{BlockAccountUpdate, BlockHeader, BlockInputs, BlockNoteTree, BlockNumber, Blockchain, ProvenBlock}; +use miden_objects::block::{ + BlockHeader, + BlockInputs, + BlockNoteTree, + BlockNumber, + Blockchain, + ProvenBlock, +}; use miden_objects::crypto::merkle::{ Forest, LargeSmt, @@ -147,9 +155,8 @@ impl State { let chain_mmr = load_mmr(&mut db).await?; let block_headers = db.select_all_block_headers().await?; - let latest_block_num = block_headers - .last() - .map_or(BlockNumber::GENESIS, BlockHeader::block_num); + let latest_block_num = + block_headers.last().map_or(BlockNumber::GENESIS, BlockHeader::block_num); let account_tree = load_account_tree(&mut db, latest_block_num).await?; let nullifier_tree = load_nullifier_tree(&mut db).await?; @@ -344,9 +351,8 @@ impl State { .output_notes() .map(|(note_index, note)| (note_index, note.id(), *note.metadata())), ); - let note_tree = - BlockNoteTree::with_entries(note_tree_entries.iter().copied()) - .map_err(|e| InvalidBlockError::FailedToBuildNoteTree(e.to_string()))?; + let note_tree = BlockNoteTree::with_entries(note_tree_entries.iter().copied()) + .map_err(|e| InvalidBlockError::FailedToBuildNoteTree(e.to_string()))?; if note_tree.root() != header.note_root() { return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); } @@ -388,15 +394,14 @@ impl State { // Signals the write lock has been acquired, and the transaction can be committed let (inform_acquire_done, acquire_done) = oneshot::channel::<()>(); - // Extract account IDs before block is moved into async task - // We'll need these later to populate the SmtForest - let updated_account_ids = Vec::::from_iter( - block - .body() - .updated_accounts() - .iter() - .map(BlockAccountUpdate::account_id), - ); + // Extract account updates with deltas before block is moved into async task + // We'll use these deltas to update the SmtForest without DB roundtrips + let account_updates: Vec<_> = block + .body() + .updated_accounts() + .iter() + .map(|update| (update.account_id(), update.details().clone())) + .collect(); // The DB and in-memory state updates need to be synchronized and are partially // overlapping. Namely, the DB transaction only proceeds after this task acquires the @@ -457,16 +462,79 @@ impl State { inner.blockchain.push(block_commitment); } - // After successful DB commit, query updated accounts' storage as well as vault data - // TODO look into making this consume the `account_tree_update` - self.update_storage_forest_from_db(updated_account_ids, block_num).await?; + // After successful DB commit, update the SmtForest with account deltas + // This uses the deltas directly without DB roundtrips, which is more efficient + self.update_forest_with_deltas(account_updates, block_num).await?; info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); Ok(()) } - /// Updates `SmtForest` after a block is successfully applied + /// Updates `SmtForest` with account deltas from a block + /// + /// This method updates the forest directly using the deltas extracted from the block, + /// avoiding database roundtrips. This is more efficient than the legacy DB-based approach. + /// + /// # Arguments + /// + /// * `account_updates` - Vector of (`AccountId`, `AccountUpdateDetails`) tuples from the block + /// * `block_num` - Block number for which these updates apply + /// + /// # Note + /// + /// - Private account updates are skipped as their state is not publicly visible + /// - Only accounts with deltas (not Private) are processed + /// - The number of changed accounts is bounded by transactions per block + #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = account_updates.len()))] + async fn update_forest_with_deltas( + &self, + account_updates: Vec<(AccountId, AccountUpdateDetails)>, + block_num: BlockNumber, + ) -> Result<(), ApplyBlockError> { + if account_updates.is_empty() { + return Ok(()); + } + + let mut forest_guard = self.forest.write().await; + + for (account_id, details) in account_updates { + match details { + AccountUpdateDetails::Delta(delta) => { + // Update the forest with vault and storage deltas + forest_guard.update_account( + block_num, + account_id, + delta.vault(), + delta.storage(), + ); + + tracing::debug!( + target: COMPONENT, + %account_id, + %block_num, + "Updated forest with account delta" + ); + }, + AccountUpdateDetails::Private => { + // Private accounts don't expose their state changes + tracing::trace!( + target: COMPONENT, + %account_id, + %block_num, + "Skipping private account update" + ); + }, + } + } + + Ok(()) + } + + /// Updates `SmtForest` from database state (DB-based) + /// + /// This method is used during initial `State::load()` where deltas are not available. + /// For block application, prefer `update_forest_with_deltas` which uses deltas directly. /// /// Must be called after the DB transaction commits successfully, so we can safely /// query the newly committed storage data. From 4bfee3034979c409cc90f429a29afe952a1bb06e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 19 Dec 2025 18:18:13 +0100 Subject: [PATCH 53/95] refactor, simplify --- crates/store/src/inner_forest.rs | 239 ++++--------------------- crates/store/src/inner_forest/tests.rs | 136 ++++++++++++++ crates/store/src/state.rs | 146 ++++++--------- 3 files changed, 225 insertions(+), 296 deletions(-) create mode 100644 crates/store/src/inner_forest/tests.rs diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index 8dda1d25a9..4b9394096c 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -9,7 +9,7 @@ use miden_objects::{EMPTY_WORD, Word}; // Type aliases to reduce complexity type MapSlotEntries = Vec<(Word, Word)>; -type StorageMapSlot = (AccountId, StorageSlotName, MapSlotEntries); + type VaultEntries = Vec<(Word, Word)>; /// Container for forest-related state that needs to be updated atomically. @@ -190,29 +190,20 @@ impl InnerForest { } } - // LEGACY DB-BASED POPULATION METHODS - // ================================================================================================ - // These methods are used during initial State::load() where deltas are not available. - // They populate the forest from full database state rather than incremental deltas. - // - // For block application, prefer `update_account()` which uses deltas directly. - - /// Populates storage map SMTs in the forest from full database state. - /// - /// **DEPRECATED for block application**: Use `update_account()` with deltas instead. - /// This method is primarily used during `State::load()` where deltas are not available. + /// Populates storage map SMTs in the forest from full database state for a single account. /// /// # Arguments /// - /// * `map_slots_to_populate` - List of (`account_id`, `slot_name`, entries) tuples + /// * `account_id` - The account whose storage maps are being initialized + /// * `map_slots_to_populate` - List of `(slot_name, entries)` tuples /// * `block_num` - Block number for which this state applies - #[allow(dead_code)] // Used only during State::load - pub(crate) fn populate_storage_maps( + pub(crate) fn add_storage_map( &mut self, - map_slots_to_populate: Vec, + account_id: AccountId, + map_slots_to_populate: Vec<(StorageSlotName, MapSlotEntries)>, block_num: BlockNumber, ) { - for (account_id, slot_name, entries) in map_slots_to_populate { + for (slot_name, entries) in map_slots_to_populate { if entries.is_empty() { continue; } @@ -236,209 +227,39 @@ impl InnerForest { } } - /// Populates vault SMTs in the forest from full database state. - /// - /// **DEPRECATED for block application**: Use `update_account()` with deltas instead. - /// This method is primarily used during `State::load()` where deltas are not available. + /// Populates a vault SMT in the forest from full database state. /// /// # Arguments /// - /// * `vault_entries_to_populate` - List of (`account_id`, `vault_entries`) tuples where entries - /// are (key, value) Word pairs + /// * `account_id` - The account whose vault is being initialized + /// * `vault_entries` - (key, value) Word pairs for the vault /// * `block_num` - Block number for which this state applies - #[allow(dead_code)] // Used only during State::load - pub(crate) fn populate_vaults( + pub(crate) fn add_vault( &mut self, - vault_entries_to_populate: Vec<(AccountId, VaultEntries)>, + account_id: AccountId, + vault_entries: VaultEntries, block_num: BlockNumber, ) { - for (account_id, entries) in vault_entries_to_populate { - if entries.is_empty() { - continue; - } - - let updated_root = self - .storage_forest - .batch_insert(Self::empty_smt_root(), entries.iter().copied()) - .expect("Forest insertion should succeed"); - - self.vault_roots.insert((account_id, block_num), updated_root); - - tracing::debug!( - target: crate::COMPONENT, - account_id = %account_id, - block_num = %block_num, - vault_entries = entries.len(), - "Populated vault in forest from DB" - ); + if vault_entries.is_empty() { + return; } - } - /// Helper method to extract storage map slots from `AccountStorage` objects. - /// - /// Used by the legacy DB-based population path during `State::load()`. - /// - /// # Returns - /// - /// Vector of (`account_id`, `slot_name`, entries) tuples ready for forest population - pub(crate) fn extract_map_slots_from_storage( - account_storages: &[(AccountId, miden_objects::account::AccountStorage)], - ) -> Vec { - use miden_objects::account::StorageSlotContent; - - let mut map_slots = Vec::new(); - - for (account_id, storage) in account_storages { - for slot in storage.slots() { - if let StorageSlotContent::Map(map) = slot.content() { - let entries: Vec<_> = map.entries().map(|(k, v)| (*k, *v)).collect(); - - if !entries.is_empty() { - map_slots.push((*account_id, slot.name().clone(), entries)); - } - } - } - } + let updated_root = self + .storage_forest + .batch_insert(Self::empty_smt_root(), vault_entries.iter().copied()) + .expect("Forest insertion should succeed"); - map_slots - } -} + self.vault_roots.insert((account_id, block_num), updated_root); -#[cfg(test)] -mod tests { - use super::*; - use miden_objects::asset::{Asset, FungibleAsset}; - use miden_objects::testing::account_id::{ - ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET, - ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE, - }; - - fn test_account() -> AccountId { - AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap() - } - - fn test_faucet() -> AccountId { - AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap() - } - - fn create_fungible_asset(faucet_id: AccountId, amount: u64) -> Asset { - FungibleAsset::new(faucet_id, amount).unwrap().into() - } - - #[test] - fn test_empty_smt_root_is_recognized() { - use miden_objects::crypto::merkle::Smt; - - let empty_root = InnerForest::empty_smt_root(); - - // Verify an empty SMT has the expected root - assert_eq!(Smt::default().root(), empty_root); - - // Test that SmtForest accepts this root in batch_insert - let mut forest = SmtForest::new(); - let entries = vec![(Word::from([1u32, 2, 3, 4]), Word::from([5u32, 6, 7, 8]))]; - - assert!(forest.batch_insert(empty_root, entries).is_ok()); - } - - #[test] - fn test_inner_forest_basic_initialization() { - let forest = InnerForest::new(); - assert!(forest.storage_roots.is_empty()); - assert!(forest.vault_roots.is_empty()); - } - - #[test] - fn test_update_account_with_empty_deltas() { - let mut forest = InnerForest::new(); - let account_id = test_account(); - let block_num = BlockNumber::GENESIS.child(); - - let vault_delta = AccountVaultDelta::default(); - let storage_delta = AccountStorageDelta::default(); - - forest.update_account(block_num, account_id, &vault_delta, &storage_delta); - - // Empty deltas should not create entries - assert!(!forest.vault_roots.contains_key(&(account_id, block_num))); - assert!(forest.storage_roots.is_empty()); - } - - #[test] - fn test_update_vault_with_fungible_asset() { - let mut forest = InnerForest::new(); - let account_id = test_account(); - let faucet_id = test_faucet(); - let block_num = BlockNumber::GENESIS.child(); - - let asset = create_fungible_asset(faucet_id, 100); - let mut vault_delta = AccountVaultDelta::default(); - vault_delta.add_asset(asset).unwrap(); - - forest.update_account(block_num, account_id, &vault_delta, &AccountStorageDelta::default()); - - let vault_root = forest.vault_roots[&(account_id, block_num)]; - assert_ne!(vault_root, EMPTY_WORD); - } - - #[test] - fn test_compare_delta_vs_db_vault_with_fungible_asset() { - let account_id = test_account(); - let faucet_id = test_faucet(); - let block_num = BlockNumber::GENESIS.child(); - let asset = create_fungible_asset(faucet_id, 100); - - // Approach 1: Delta-based update - let mut forest_delta = InnerForest::new(); - let mut vault_delta = AccountVaultDelta::default(); - vault_delta.add_asset(asset).unwrap(); - forest_delta.update_account( - block_num, - account_id, - &vault_delta, - &AccountStorageDelta::default(), + tracing::debug!( + target: crate::COMPONENT, + account_id = %account_id, + block_num = %block_num, + vault_entries = vault_entries.len(), + "Populated vault in forest from DB" ); - - // Approach 2: DB-based population - let mut forest_db = InnerForest::new(); - let vault_entries = vec![(asset.vault_key().into(), Word::from(asset))]; - forest_db.populate_vaults(vec![(account_id, vault_entries)], block_num); - - // Both approaches must produce identical roots - let root_delta = forest_delta.vault_roots.get(&(account_id, block_num)).unwrap(); - let root_db = forest_db.vault_roots.get(&(account_id, block_num)).unwrap(); - - assert_eq!(root_delta, root_db); - assert_ne!(*root_delta, EMPTY_WORD); - } - - #[test] - fn test_slot_names_are_tracked() { - let forest = InnerForest::new(); - let _: &BTreeMap<(AccountId, StorageSlotName, BlockNumber), Word> = &forest.storage_roots; - } - - #[test] - fn test_incremental_vault_updates() { - let mut forest = InnerForest::new(); - let account_id = test_account(); - let faucet_id = test_faucet(); - let storage_delta = AccountStorageDelta::default(); - - // Block 1: 100 tokens - let block_1 = BlockNumber::GENESIS.child(); - let mut vault_delta_1 = AccountVaultDelta::default(); - vault_delta_1.add_asset(create_fungible_asset(faucet_id, 100)).unwrap(); - forest.update_account(block_1, account_id, &vault_delta_1, &storage_delta); - let root_1 = forest.vault_roots[&(account_id, block_1)]; - - // Block 2: 150 tokens - let block_2 = block_1.child(); - let mut vault_delta_2 = AccountVaultDelta::default(); - vault_delta_2.add_asset(create_fungible_asset(faucet_id, 150)).unwrap(); - forest.update_account(block_2, account_id, &vault_delta_2, &storage_delta); - let root_2 = forest.vault_roots[&(account_id, block_2)]; - - assert_ne!(root_1, root_2); } } + +#[cfg(test)] +mod tests; diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs new file mode 100644 index 0000000000..8165fc4b59 --- /dev/null +++ b/crates/store/src/inner_forest/tests.rs @@ -0,0 +1,136 @@ +use miden_objects::asset::{Asset, FungibleAsset}; +use miden_objects::testing::account_id::{ + ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET, + ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE, +}; + +use super::*; + +fn test_account() -> AccountId { + AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap() +} + +fn test_faucet() -> AccountId { + AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap() +} + +fn create_fungible_asset(faucet_id: AccountId, amount: u64) -> Asset { + FungibleAsset::new(faucet_id, amount).unwrap().into() +} + +#[test] +fn test_empty_smt_root_is_recognized() { + use miden_objects::crypto::merkle::Smt; + + let empty_root = InnerForest::empty_smt_root(); + + // Verify an empty SMT has the expected root + assert_eq!(Smt::default().root(), empty_root); + + // Test that SmtForest accepts this root in batch_insert + let mut forest = SmtForest::new(); + let entries = vec![(Word::from([1u32, 2, 3, 4]), Word::from([5u32, 6, 7, 8]))]; + + assert!(forest.batch_insert(empty_root, entries).is_ok()); +} + +#[test] +fn test_inner_forest_basic_initialization() { + let forest = InnerForest::new(); + assert!(forest.storage_roots.is_empty()); + assert!(forest.vault_roots.is_empty()); +} + +#[test] +fn test_update_account_with_empty_deltas() { + let mut forest = InnerForest::new(); + let account_id = test_account(); + let block_num = BlockNumber::GENESIS.child(); + + let vault_delta = AccountVaultDelta::default(); + let storage_delta = AccountStorageDelta::default(); + + forest.update_account(block_num, account_id, &vault_delta, &storage_delta); + + // Empty deltas should not create entries + assert!(!forest.vault_roots.contains_key(&(account_id, block_num))); + assert!(forest.storage_roots.is_empty()); +} + +#[test] +fn test_update_vault_with_fungible_asset() { + let mut forest = InnerForest::new(); + let account_id = test_account(); + let faucet_id = test_faucet(); + let block_num = BlockNumber::GENESIS.child(); + + let asset = create_fungible_asset(faucet_id, 100); + let mut vault_delta = AccountVaultDelta::default(); + vault_delta.add_asset(asset).unwrap(); + + forest.update_account(block_num, account_id, &vault_delta, &AccountStorageDelta::default()); + + let vault_root = forest.vault_roots[&(account_id, block_num)]; + assert_ne!(vault_root, EMPTY_WORD); +} + +#[test] +fn test_compare_delta_vs_db_vault_with_fungible_asset() { + let account_id = test_account(); + let faucet_id = test_faucet(); + let block_num = BlockNumber::GENESIS.child(); + let asset = create_fungible_asset(faucet_id, 100); + + // Approach 1: Delta-based update + let mut forest_delta = InnerForest::new(); + let mut vault_delta = AccountVaultDelta::default(); + vault_delta.add_asset(asset).unwrap(); + forest_delta.update_account( + block_num, + account_id, + &vault_delta, + &AccountStorageDelta::default(), + ); + + // Approach 2: DB-based population + let mut forest_db = InnerForest::new(); + let vault_entries = vec![(asset.vault_key().into(), Word::from(asset))]; + forest_db.add_vault(account_id, vault_entries, block_num); + + // Both approaches must produce identical roots + let root_delta = forest_delta.vault_roots.get(&(account_id, block_num)).unwrap(); + let root_db = forest_db.vault_roots.get(&(account_id, block_num)).unwrap(); + + assert_eq!(root_delta, root_db); + assert_ne!(*root_delta, EMPTY_WORD); +} + +#[test] +fn test_slot_names_are_tracked() { + let forest = InnerForest::new(); + let _: &BTreeMap<(AccountId, StorageSlotName, BlockNumber), Word> = &forest.storage_roots; +} + +#[test] +fn test_incremental_vault_updates() { + let mut forest = InnerForest::new(); + let account_id = test_account(); + let faucet_id = test_faucet(); + let storage_delta = AccountStorageDelta::default(); + + // Block 1: 100 tokens + let block_1 = BlockNumber::GENESIS.child(); + let mut vault_delta_1 = AccountVaultDelta::default(); + vault_delta_1.add_asset(create_fungible_asset(faucet_id, 100)).unwrap(); + forest.update_account(block_1, account_id, &vault_delta_1, &storage_delta); + let root_1 = forest.vault_roots[&(account_id, block_1)]; + + // Block 2: 150 tokens + let block_2 = block_1.child(); + let mut vault_delta_2 = AccountVaultDelta::default(); + vault_delta_2.add_asset(create_fungible_asset(faucet_id, 150)).unwrap(); + forest.update_account(block_2, account_id, &vault_delta_2, &storage_delta); + let root_2 = forest.vault_roots[&(account_id, block_2)]; + + assert_ne!(root_1, root_2); +} diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 048b9c8c89..0efeea3a2a 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -24,7 +24,7 @@ use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; use miden_objects::account::delta::AccountUpdateDetails; -use miden_objects::account::{AccountId, AccountStorage, StorageSlotContent}; +use miden_objects::account::{AccountId, AccountStorage, StorageSlotContent, StorageSlotName}; use miden_objects::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_objects::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_objects::block::{ @@ -180,7 +180,7 @@ impl State { let acc_account_ids = me.db.select_all_account_commitments().await?; let acc_account_ids = Vec::from_iter(acc_account_ids.into_iter().map(|(account_id, _)| account_id)); - me.update_storage_forest_from_db(acc_account_ids, latest_block_num) + me.initialize_storage_forest_from_db(acc_account_ids, latest_block_num) .await .map_err(|e| { StateInitializationError::DatabaseError(DatabaseError::InteractError(format!( @@ -464,7 +464,7 @@ impl State { // After successful DB commit, update the SmtForest with account deltas // This uses the deltas directly without DB roundtrips, which is more efficient - self.update_forest_with_deltas(account_updates, block_num).await?; + self.update_forest(account_updates, block_num).await?; info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); @@ -473,8 +473,7 @@ impl State { /// Updates `SmtForest` with account deltas from a block /// - /// This method updates the forest directly using the deltas extracted from the block, - /// avoiding database roundtrips. This is more efficient than the legacy DB-based approach. + /// This method updates the forest directly using the deltas extracted from the block. /// /// # Arguments /// @@ -483,11 +482,11 @@ impl State { /// /// # Note /// - /// - Private account updates are skipped as their state is not publicly visible - /// - Only accounts with deltas (not Private) are processed - /// - The number of changed accounts is bounded by transactions per block + /// - Private account updates are skipped as their state is not publicly visible. + /// - The number of changed accounts is implicitly bounded by the limited number of transactions + /// per block. #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = account_updates.len()))] - async fn update_forest_with_deltas( + async fn update_forest( &self, account_updates: Vec<(AccountId, AccountUpdateDetails)>, block_num: BlockNumber, @@ -534,102 +533,47 @@ impl State { /// Updates `SmtForest` from database state (DB-based) /// /// This method is used during initial `State::load()` where deltas are not available. - /// For block application, prefer `update_forest_with_deltas` which uses deltas directly. - /// - /// Must be called after the DB transaction commits successfully, so we can safely - /// query the newly committed storage data. + /// For block application, prefer `fn update_forest` which uses deltas directly. /// /// # Warning /// /// Has internal locking to mutate the state, use cautiously in scopes with other /// mutex guards around! - /// - /// # Note - /// - /// The number of changed accounts is bounded by transactions per block. - async fn update_storage_forest_from_db( + #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num))] + async fn initialize_storage_forest_from_db( &self, - changed_account_ids: Vec, - block_num: BlockNumber, - ) -> Result<(), ApplyBlockError> { - if changed_account_ids.is_empty() { - return Ok(()); - } - - self.update_storage_maps_in_forest(&changed_account_ids, block_num).await?; - - self.update_vaults_in_forest(&changed_account_ids, block_num).await?; - - Ok(()) - } - - /// Updates storage map SMTs in the forest for changed accounts - #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = changed_account_ids.len()))] - async fn update_storage_maps_in_forest( - &self, - changed_account_ids: &[AccountId], + account_ids: Vec, block_num: BlockNumber, ) -> Result<(), ApplyBlockError> { - // Step 1: Query storage from database - let account_storages = - self.query_account_storages_from_db(changed_account_ids, block_num).await?; - - // Step 2: Extract map slots and their entries using InnerForest helper - let map_slots_to_populate = InnerForest::extract_map_slots_from_storage(&account_storages); - - if map_slots_to_populate.is_empty() { - return Ok(()); - } - - // Step 3: Acquire write lock and update the forest with new SMTs + // Acquire write lock once for the entire initialization let mut forest_guard = self.forest.write().await; - forest_guard.populate_storage_maps(map_slots_to_populate, block_num); - - Ok(()) - } - - /// Queries account storage data from the database for the given accounts at a specific block - #[instrument(target = COMPONENT, skip_all, fields(num_accounts = account_ids.len()))] - async fn query_account_storages_from_db( - &self, - account_ids: &[AccountId], - block_num: BlockNumber, - ) -> Result, ApplyBlockError> { - let mut account_storages = Vec::with_capacity(account_ids.len()); - for &account_id in account_ids { + // Process each account, updating both storage maps and vaults + for account_id in account_ids { + // Query and update storage maps for this account let storage = self.db.select_account_storage_at_block(account_id, block_num).await?; - account_storages.push((account_id, storage)); - } + let map_slots = extract_map_slots_from_storage(&storage); - Ok(account_storages) - } + if !map_slots.is_empty() { + forest_guard.add_storage_map(account_id, map_slots, block_num); + } - /// Updates vault SMTs in the forest for changed accounts - #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = changed_account_ids.len()))] - async fn update_vaults_in_forest( - &self, - changed_account_ids: &[AccountId], - block_num: BlockNumber, - ) -> Result<(), ApplyBlockError> { - // Query vault assets for each updated account - let mut vault_entries_to_populate = Vec::new(); + // Query and update vault for this account + let vault_entries = + self.db.select_account_vault_at_block(account_id, block_num).await?; - for &account_id in changed_account_ids { - let entries = self.db.select_account_vault_at_block(account_id, block_num).await?; - if !entries.is_empty() { - vault_entries_to_populate.push((account_id, entries)); + if !vault_entries.is_empty() { + forest_guard.add_vault(account_id, vault_entries, block_num); } - } - if vault_entries_to_populate.is_empty() { - return Ok(()); + tracing::debug!( + target: COMPONENT, + %account_id, + %block_num, + "Initialized forest for account from DB" + ); } - // Acquire write lock once for the entire update operation and delegate to InnerForest - let mut forest_guard = self.forest.write().await; - forest_guard.populate_vaults(vault_entries_to_populate, block_num); - Ok(()) } @@ -1453,3 +1397,31 @@ async fn load_account_tree( Ok(AccountTreeWithHistory::new(account_tree, block_number)) } + +// HELPERS +// ================================================================================================= + +/// Extract storage map slots from a single `AccountStorage` object. +/// +/// # Returns +/// +/// Vector of `(account_id, slot_name, entries)` tuples ready for forest population. +pub(crate) fn extract_map_slots_from_storage( + storage: &miden_objects::account::AccountStorage, +) -> Vec<(StorageSlotName, Vec<(Word, Word)>)> { + use miden_objects::account::StorageSlotContent; + + let mut map_slots = Vec::new(); + + for slot in storage.slots() { + if let StorageSlotContent::Map(map) = slot.content() { + let entries = Vec::from_iter(map.entries().map(|(k, v)| (*k, *v))); + + if !entries.is_empty() { + map_slots.push((slot.name().clone(), entries)); + } + } + } + + map_slots +} From b8d2e66f7b3a8895b7253f7fb7a4e7f6b52532af Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 19 Dec 2025 18:23:45 +0100 Subject: [PATCH 54/95] yuk --- crates/store/src/inner_forest.rs | 9 ++++----- crates/store/src/inner_forest/tests.rs | 2 +- crates/store/src/state.rs | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index 4b9394096c..0e53e2c32d 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -7,7 +7,9 @@ use miden_objects::block::BlockNumber; use miden_objects::crypto::merkle::{EmptySubtreeRoots, SMT_DEPTH, SmtForest}; use miden_objects::{EMPTY_WORD, Word}; -// Type aliases to reduce complexity +#[cfg(test)] +mod tests; + type MapSlotEntries = Vec<(Word, Word)>; type VaultEntries = Vec<(Word, Word)>; @@ -237,7 +239,7 @@ impl InnerForest { pub(crate) fn add_vault( &mut self, account_id: AccountId, - vault_entries: VaultEntries, + vault_entries: &VaultEntries, block_num: BlockNumber, ) { if vault_entries.is_empty() { @@ -260,6 +262,3 @@ impl InnerForest { ); } } - -#[cfg(test)] -mod tests; diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs index 8165fc4b59..4de7f3808e 100644 --- a/crates/store/src/inner_forest/tests.rs +++ b/crates/store/src/inner_forest/tests.rs @@ -95,7 +95,7 @@ fn test_compare_delta_vs_db_vault_with_fungible_asset() { // Approach 2: DB-based population let mut forest_db = InnerForest::new(); let vault_entries = vec![(asset.vault_key().into(), Word::from(asset))]; - forest_db.add_vault(account_id, vault_entries, block_num); + forest_db.add_vault(account_id, &vault_entries, block_num); // Both approaches must produce identical roots let root_delta = forest_delta.vault_roots.get(&(account_id, block_num)).unwrap(); diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 0efeea3a2a..e1ca0bb1e8 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -563,7 +563,7 @@ impl State { self.db.select_account_vault_at_block(account_id, block_num).await?; if !vault_entries.is_empty() { - forest_guard.add_vault(account_id, vault_entries, block_num); + forest_guard.add_vault(account_id, &vault_entries, block_num); } tracing::debug!( From e453faa1f6e7de71ecd55ffa711afe2e6ab4d848 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 19 Dec 2025 19:21:54 +0100 Subject: [PATCH 55/95] remoe useless comment --- crates/store/src/db/models/queries/accounts.rs | 9 ++------- crates/store/src/state.rs | 4 ---- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 7457591329..28da8a3ef2 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -45,18 +45,13 @@ use crate::db::{AccountVaultValue, schema}; use crate::errors::DatabaseError; type StorageMapValueRow = (i64, String, Vec, Vec); -/// [`SqliteConnection`]. + +/// Select account by ID from the DB using the given [`SqliteConnection`]. /// /// # Returns /// /// The latest account info, or an error. /// -/// # Note -/// -/// Returns only the account summary. Full account details must be reconstructed -/// in follow up query, using separate query functions to fetch specific account -/// components as needed. -/// /// # Raw SQL /// /// ```sql diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index e1ca0bb1e8..2003948de8 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -175,8 +175,6 @@ impl State { let me = Self { db, block_store, inner, forest, writer }; // load all accounts from the table - // TODO: make `select_all_account_at(block_num)` to be precise; if ACID is upheld, it's not - // necessary in theory let acc_account_ids = me.db.select_all_account_commitments().await?; let acc_account_ids = Vec::from_iter(acc_account_ids.into_iter().map(|(account_id, _)| account_id)); @@ -462,8 +460,6 @@ impl State { inner.blockchain.push(block_commitment); } - // After successful DB commit, update the SmtForest with account deltas - // This uses the deltas directly without DB roundtrips, which is more efficient self.update_forest(account_updates, block_num).await?; info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); From f6d1ce1c74e3b9ef85f1ed6d5630b9397e4035fe Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 19 Dec 2025 19:35:46 +0100 Subject: [PATCH 56/95] shorthandg pu --- crates/proto/src/domain/account.rs | 33 ++++++++++++++---------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 60c840154b..2a36614d9e 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -193,6 +193,7 @@ impl TryFrom fn try_from( value: proto::rpc::account_storage_details::AccountStorageMapDetails, ) -> Result { + use proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry; let proto::rpc::account_storage_details::AccountStorageMapDetails { slot_name, too_many_entries, @@ -206,24 +207,20 @@ impl TryFrom } else { let map_entries = if let Some(entries) = entries { entries - .entries -.into_iter() -.map(|entry| { -let key = entry -.key - .ok_or(proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry::missing_field( - stringify!(key), - ))? - .try_into()?; -let value = entry -.value - .ok_or(proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry::missing_field( - stringify!(value), - ))? - .try_into()?; - Ok((key, value)) - }) - .collect::, ConversionError>>()? + .entries + .into_iter() + .map(|entry| { + let key = entry + .key + .ok_or(StorageMapEntry::missing_field(stringify!(key)))? + .try_into()?; + let value = entry + .value + .ok_or(StorageMapEntry::missing_field(stringify!(value)))? + .try_into()?; + Ok((key, value)) + }) + .collect::, ConversionError>>()? } else { Vec::new() }; From b1f9cf6f44c793d1291317a2d4215d722471dbdb Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 19 Dec 2025 20:32:04 +0100 Subject: [PATCH 57/95] delete unused --- crates/store/src/db/mod.rs | 16 +--------------- crates/store/src/state.rs | 26 +------------------------- 2 files changed, 2 insertions(+), 40 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 704f70a6e4..586de2206e 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -322,7 +322,7 @@ impl Db { /// Loads all the nullifiers from the DB. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub async fn select_all_nullifiers(&self) -> Result> { + pub(crate) async fn select_all_nullifiers(&self) -> Result> { self.transact("all nullifiers", move |conn| { let nullifiers = queries::select_all_nullifiers(conn)?; Ok(nullifiers) @@ -443,20 +443,6 @@ impl Db { .await } - /// Gets the latest account storage from the database - /// - /// Uses the `is_latest` flag for efficient querying. - #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub async fn select_latest_account_storage( - &self, - account_id: AccountId, - ) -> Result { - self.transact("Get latest account storage", move |conn| { - queries::select_latest_account_storage(conn, account_id) - }) - .await - } - /// Queries vault assets at a specific block #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_account_vault_at_block( diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 2003948de8..c58e33914d 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -24,7 +24,7 @@ use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; use miden_objects::account::delta::AccountUpdateDetails; -use miden_objects::account::{AccountId, AccountStorage, StorageSlotContent, StorageSlotName}; +use miden_objects::account::{AccountId, StorageSlotContent, StorageSlotName}; use miden_objects::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_objects::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_objects::block::{ @@ -1061,30 +1061,6 @@ impl State { self.db.select_network_account_by_prefix(id_prefix).await } - /// Reconstructs account storage at a specific block - /// - /// # Errors - /// - /// Returns an error if the block doesn't exist or if there's a database error. - pub async fn get_account_storage_at_block( - &self, - account_id: AccountId, - block_num: BlockNumber, - ) -> Result { - // Validate block exists in the blockchain before querying the database - self.validate_block_exists(block_num).await?; - - self.db.select_account_storage_at_block(account_id, block_num).await - } - - /// Gets the latest account storage - pub async fn get_latest_account_storage( - &self, - account_id: AccountId, - ) -> Result { - self.db.select_latest_account_storage(account_id).await - } - /// Returns account IDs for all public (on-chain) network accounts. pub async fn get_all_network_accounts(&self) -> Result, DatabaseError> { self.db.select_all_network_account_ids().await From d2d9e8c6abaef5304eb5ad7ff301bcdce359278e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Sat, 20 Dec 2025 14:37:53 +0100 Subject: [PATCH 58/95] review --- crates/proto/src/domain/account.rs | 9 +++++++++ crates/store/src/db/mod.rs | 2 +- crates/store/src/db/models/queries/accounts.rs | 18 +++++++++--------- crates/store/src/inner_forest.rs | 18 +++++++++++------- crates/store/src/inner_forest/tests.rs | 4 ++-- crates/store/src/state.rs | 13 +++++-------- 6 files changed, 37 insertions(+), 27 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 2a36614d9e..feb527f775 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -381,6 +381,15 @@ impl AccountVaultDetails { Self::Assets(Vec::new()) } + /// Creates `AccountVaultDetails` from a list of assets. + pub fn from_assets(assets: Vec) -> Self { + if assets.len() > Self::MAX_RETURN_ENTRIES { + Self::LimitExceeded + } else { + Self::Assets(assets) + } + } + /// Creates `AccountVaultDetails` from vault entries (key-value pairs). /// /// This is useful when entries have been fetched directly from the database diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 586de2206e..7371094684 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -449,7 +449,7 @@ impl Db { &self, account_id: AccountId, block_num: BlockNumber, - ) -> Result> { + ) -> Result> { self.transact("Get account vault at block", move |conn| { queries::select_account_vault_at_block(conn, account_id, block_num) }) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 28da8a3ef2..0a88d8e37c 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -989,7 +989,7 @@ pub(crate) fn select_account_vault_at_block( conn: &mut SqliteConnection, account_id: AccountId, block_num: BlockNumber, -) -> Result, DatabaseError> { +) -> Result, DatabaseError> { use schema::account_vault_assets as t; let account_id_bytes = account_id.to_bytes(); @@ -1015,29 +1015,29 @@ pub(crate) fn select_account_vault_at_block( } // Step 2: Fetch the full rows matching (vault_key, block_num) pairs - let mut entries = Vec::new(); + let mut assets = Vec::new(); for (vault_key_bytes, max_block) in latest_blocks_per_vault_key { - let result: Option<(Vec, Option>)> = QueryDsl::select( + let result: Option>> = QueryDsl::select( t::table.filter( t::account_id .eq(&account_id_bytes) .and(t::vault_key.eq(&vault_key_bytes)) .and(t::block_num.eq(max_block)), ), - (t::vault_key, t::asset), + t::asset, ) .first(conn) .optional()?; - if let Some((key_bytes, Some(asset_bytes))) = result { - entries - .push((Word::read_from_bytes(&key_bytes)?, Word::read_from_bytes(&asset_bytes)?)); + if let Some(Some(asset_bytes)) = result { + let asset = Asset::read_from_bytes(&asset_bytes)?; + assets.push(asset); } } // Sort by vault_key for consistent ordering - entries.sort_by_key(|(key, _)| *key); + assets.sort_by_key(Asset::vault_key); - Ok(entries) + Ok(assets) } /// Queries the account code for a specific account at a specific block number. diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index 0e53e2c32d..bebf4dce43 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -12,8 +12,6 @@ mod tests; type MapSlotEntries = Vec<(Word, Word)>; -type VaultEntries = Vec<(Word, Word)>; - /// Container for forest-related state that needs to be updated atomically. pub(crate) struct InnerForest { /// `SmtForest` for efficient account storage reconstruction. @@ -234,21 +232,27 @@ impl InnerForest { /// # Arguments /// /// * `account_id` - The account whose vault is being initialized - /// * `vault_entries` - (key, value) Word pairs for the vault + /// * `assets` - Assets to populate the vault with /// * `block_num` - Block number for which this state applies pub(crate) fn add_vault( &mut self, account_id: AccountId, - vault_entries: &VaultEntries, + assets: &[Asset], block_num: BlockNumber, ) { - if vault_entries.is_empty() { + if assets.is_empty() { return; } + // Convert assets to (key, value) pairs for SMT insertion + let entries: Vec<(Word, Word)> = assets + .iter() + .map(|asset| (asset.vault_key().into(), Word::from(*asset))) + .collect(); + let updated_root = self .storage_forest - .batch_insert(Self::empty_smt_root(), vault_entries.iter().copied()) + .batch_insert(Self::empty_smt_root(), entries.iter().copied()) .expect("Forest insertion should succeed"); self.vault_roots.insert((account_id, block_num), updated_root); @@ -257,7 +261,7 @@ impl InnerForest { target: crate::COMPONENT, account_id = %account_id, block_num = %block_num, - vault_entries = vault_entries.len(), + vault_entries = assets.len(), "Populated vault in forest from DB" ); } diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs index 4de7f3808e..993b49fd28 100644 --- a/crates/store/src/inner_forest/tests.rs +++ b/crates/store/src/inner_forest/tests.rs @@ -94,8 +94,8 @@ fn test_compare_delta_vs_db_vault_with_fungible_asset() { // Approach 2: DB-based population let mut forest_db = InnerForest::new(); - let vault_entries = vec![(asset.vault_key().into(), Word::from(asset))]; - forest_db.add_vault(account_id, &vault_entries, block_num); + let vault_assets = vec![asset]; + forest_db.add_vault(account_id, &vault_assets, block_num); // Both approaches must produce identical roots let root_delta = forest_delta.vault_roots.get(&(account_id, block_num)).unwrap(); diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index c58e33914d..569c5e5de3 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -555,11 +555,10 @@ impl State { } // Query and update vault for this account - let vault_entries = - self.db.select_account_vault_at_block(account_id, block_num).await?; + let vault_assets = self.db.select_account_vault_at_block(account_id, block_num).await?; - if !vault_entries.is_empty() { - forest_guard.add_vault(account_id, &vault_entries, block_num); + if !vault_assets.is_empty() { + forest_guard.add_vault(account_id, &vault_assets, block_num); } tracing::debug!( @@ -1166,11 +1165,9 @@ impl State { AccountVaultDetails::empty() }, Some(_) | None if asset_vault_commitment.is_some() => { - let vault_entries = + let vault_assets = self.db.select_account_vault_at_block(account_id, block_num).await?; - AccountVaultDetails::from_entries(vault_entries).map_err(|e| { - DatabaseError::InteractError(format!("Failed to parse vault assets: {e}")) - })? + AccountVaultDetails::from_assets(vault_assets) }, _ => AccountVaultDetails::empty(), }; From 2781db85b6847275ad5786903905fa8f8fc5cb10 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 22 Dec 2025 17:45:55 +0100 Subject: [PATCH 59/95] simplify --- crates/store/src/inner_forest.rs | 149 +++++++----------------- crates/store/src/inner_forest/tests.rs | 150 ++++++++++++++++++------- crates/store/src/state.rs | 83 +++++--------- 3 files changed, 182 insertions(+), 200 deletions(-) diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index bebf4dce43..abdbdd2e98 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use miden_objects::account::delta::{AccountStorageDelta, AccountVaultDelta}; +use miden_objects::account::delta::{AccountDelta, AccountStorageDelta, AccountVaultDelta}; use miden_objects::account::{AccountId, NonFungibleDeltaAction, StorageSlotName}; use miden_objects::asset::{Asset, FungibleAsset}; use miden_objects::block::BlockNumber; @@ -10,8 +10,6 @@ use miden_objects::{EMPTY_WORD, Word}; #[cfg(test)] mod tests; -type MapSlotEntries = Vec<(Word, Word)>; - /// Container for forest-related state that needs to be updated atomically. pub(crate) struct InnerForest { /// `SmtForest` for efficient account storage reconstruction. @@ -44,29 +42,29 @@ impl InnerForest { /// Updates the forest with account vault and storage changes from a delta. /// /// This is the unified interface for updating all account state in the forest. - /// It processes both vault and storage map deltas and updates the forest accordingly. + /// It handles both full-state deltas (new accounts or reconstruction from DB) + /// and partial deltas (incremental updates during block application). + /// + /// For full-state deltas (`delta.is_full_state() == true`), the forest is populated + /// from scratch using an empty SMT root. For partial deltas, changes are applied + /// on top of the previous block's state. /// /// # Arguments /// /// * `block_num` - Block number for which these changes are being applied - /// * `account_id` - The account being updated - /// * `vault_delta` - Changes to the account's asset vault - /// * `storage_delta` - Changes to the account's storage maps - pub(crate) fn update_account( - &mut self, - block_num: BlockNumber, - account_id: AccountId, - vault_delta: &AccountVaultDelta, - storage_delta: &AccountStorageDelta, - ) { + /// * `delta` - The account delta containing vault and storage changes + pub(crate) fn update_account(&mut self, block_num: BlockNumber, delta: &AccountDelta) { + let account_id = delta.id(); + let is_full_state = delta.is_full_state(); + // Update vault if there are any changes - if !vault_delta.is_empty() { - self.update_account_vault(block_num, account_id, vault_delta); + if !delta.vault().is_empty() { + self.update_account_vault(block_num, account_id, delta.vault(), is_full_state); } // Update storage maps if there are any changes - if !storage_delta.is_empty() { - self.update_account_storage(block_num, account_id, storage_delta); + if !delta.storage().is_empty() { + self.update_account_storage(block_num, account_id, delta.storage(), is_full_state); } } @@ -80,18 +78,25 @@ impl InnerForest { /// * `block_num` - Block number for this update /// * `account_id` - The account being updated /// * `vault_delta` - Changes to the account's asset vault + /// * `is_full_state` - If true, start from empty root; otherwise use previous block's root fn update_account_vault( &mut self, block_num: BlockNumber, account_id: AccountId, vault_delta: &AccountVaultDelta, + is_full_state: bool, ) { - let prev_block_num = block_num.parent().unwrap_or_default(); - let prev_root = self - .vault_roots - .get(&(account_id, prev_block_num)) - .copied() - .unwrap_or_else(Self::empty_smt_root); + // For full-state deltas (new accounts or reconstruction), start from empty root. + // For partial deltas, look up the previous block's root. + let prev_root = if is_full_state { + Self::empty_smt_root() + } else { + let prev_block_num = block_num.parent().unwrap_or_default(); + self.vault_roots + .get(&(account_id, prev_block_num)) + .copied() + .unwrap_or_else(Self::empty_smt_root) + }; // Collect all vault entry updates let mut entries = Vec::new(); @@ -147,20 +152,26 @@ impl InnerForest { /// * `block_num` - Block number for this update /// * `account_id` - The account being updated /// * `storage_delta` - Changes to the account's storage maps + /// * `is_full_state` - If true, start from empty root; otherwise use previous block's root fn update_account_storage( &mut self, block_num: BlockNumber, account_id: AccountId, storage_delta: &AccountStorageDelta, + is_full_state: bool, ) { - let prev_block_num = block_num.parent().unwrap_or_default(); - for (slot_name, map_delta) in storage_delta.maps() { - let prev_root = self - .storage_roots - .get(&(account_id, slot_name.clone(), prev_block_num)) - .copied() - .unwrap_or_else(Self::empty_smt_root); + // For full-state deltas (new accounts or reconstruction), start from empty root. + // For partial deltas, look up the previous block's root. + let prev_root = if is_full_state { + Self::empty_smt_root() + } else { + let prev_block_num = block_num.parent().unwrap_or_default(); + self.storage_roots + .get(&(account_id, slot_name.clone(), prev_block_num)) + .copied() + .unwrap_or_else(Self::empty_smt_root) + }; // Collect entries from the delta let entries = map_delta @@ -189,80 +200,4 @@ impl InnerForest { } } } - - /// Populates storage map SMTs in the forest from full database state for a single account. - /// - /// # Arguments - /// - /// * `account_id` - The account whose storage maps are being initialized - /// * `map_slots_to_populate` - List of `(slot_name, entries)` tuples - /// * `block_num` - Block number for which this state applies - pub(crate) fn add_storage_map( - &mut self, - account_id: AccountId, - map_slots_to_populate: Vec<(StorageSlotName, MapSlotEntries)>, - block_num: BlockNumber, - ) { - for (slot_name, entries) in map_slots_to_populate { - if entries.is_empty() { - continue; - } - - let updated_root = self - .storage_forest - .batch_insert(Self::empty_smt_root(), entries.iter().copied()) - .expect("Forest insertion should succeed"); - - self.storage_roots - .insert((account_id, slot_name.clone(), block_num), updated_root); - - tracing::debug!( - target: crate::COMPONENT, - account_id = %account_id, - block_num = %block_num, - slot_name = ?slot_name, - entries = entries.len(), - "Populated storage map in forest from DB" - ); - } - } - - /// Populates a vault SMT in the forest from full database state. - /// - /// # Arguments - /// - /// * `account_id` - The account whose vault is being initialized - /// * `assets` - Assets to populate the vault with - /// * `block_num` - Block number for which this state applies - pub(crate) fn add_vault( - &mut self, - account_id: AccountId, - assets: &[Asset], - block_num: BlockNumber, - ) { - if assets.is_empty() { - return; - } - - // Convert assets to (key, value) pairs for SMT insertion - let entries: Vec<(Word, Word)> = assets - .iter() - .map(|asset| (asset.vault_key().into(), Word::from(*asset))) - .collect(); - - let updated_root = self - .storage_forest - .batch_insert(Self::empty_smt_root(), entries.iter().copied()) - .expect("Forest insertion should succeed"); - - self.vault_roots.insert((account_id, block_num), updated_root); - - tracing::debug!( - target: crate::COMPONENT, - account_id = %account_id, - block_num = %block_num, - vault_entries = assets.len(), - "Populated vault in forest from DB" - ); - } } diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs index 993b49fd28..868d59871b 100644 --- a/crates/store/src/inner_forest/tests.rs +++ b/crates/store/src/inner_forest/tests.rs @@ -1,23 +1,56 @@ -use miden_objects::asset::{Asset, FungibleAsset}; +use miden_objects::account::AccountCode; +use miden_objects::asset::{Asset, AssetVault, FungibleAsset}; use miden_objects::testing::account_id::{ ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET, ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE, }; +use miden_objects::{Felt, FieldElement}; use super::*; -fn test_account() -> AccountId { +fn dummy_account() -> AccountId { AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap() } -fn test_faucet() -> AccountId { +fn dummy_faucet() -> AccountId { AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap() } -fn create_fungible_asset(faucet_id: AccountId, amount: u64) -> Asset { +fn dummy_fungible_asset(faucet_id: AccountId, amount: u64) -> Asset { FungibleAsset::new(faucet_id, amount).unwrap().into() } +/// Creates a partial `AccountDelta` (without code) for testing incremental updates. +fn dummy_partial_delta( + account_id: AccountId, + vault_delta: AccountVaultDelta, + storage_delta: AccountStorageDelta, +) -> AccountDelta { + // For partial deltas, nonce_delta must be > 0 if there are changes + let nonce_delta = if vault_delta.is_empty() && storage_delta.is_empty() { + Felt::ZERO + } else { + Felt::ONE + }; + AccountDelta::new(account_id, storage_delta, vault_delta, nonce_delta).unwrap() +} + +/// Creates a full-state `AccountDelta` (with code) for testing DB reconstruction. +fn dummy_full_state_delta(account_id: AccountId, assets: &[Asset]) -> AccountDelta { + use miden_objects::account::{Account, AccountStorage}; + + // Create a minimal account with the given assets + let vault = AssetVault::new(assets).unwrap(); + let storage = AccountStorage::new(vec![]).unwrap(); + let code = AccountCode::mock(); + let nonce = Felt::ONE; + + let account = Account::new(account_id, vault, storage, code, nonce, None).unwrap(); + + // Convert to delta - this will be a full-state delta because it has code + AccountDelta::try_from(account).unwrap() +} + #[test] fn test_empty_smt_root_is_recognized() { use miden_objects::crypto::merkle::Smt; @@ -44,13 +77,16 @@ fn test_inner_forest_basic_initialization() { #[test] fn test_update_account_with_empty_deltas() { let mut forest = InnerForest::new(); - let account_id = test_account(); + let account_id = dummy_account(); let block_num = BlockNumber::GENESIS.child(); - let vault_delta = AccountVaultDelta::default(); - let storage_delta = AccountStorageDelta::default(); + let delta = dummy_partial_delta( + account_id, + AccountVaultDelta::default(), + AccountStorageDelta::default(), + ); - forest.update_account(block_num, account_id, &vault_delta, &storage_delta); + forest.update_account(block_num, &delta); // Empty deltas should not create entries assert!(!forest.vault_roots.contains_key(&(account_id, block_num))); @@ -60,49 +96,47 @@ fn test_update_account_with_empty_deltas() { #[test] fn test_update_vault_with_fungible_asset() { let mut forest = InnerForest::new(); - let account_id = test_account(); - let faucet_id = test_faucet(); + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); let block_num = BlockNumber::GENESIS.child(); - let asset = create_fungible_asset(faucet_id, 100); + let asset = dummy_fungible_asset(faucet_id, 100); let mut vault_delta = AccountVaultDelta::default(); vault_delta.add_asset(asset).unwrap(); - forest.update_account(block_num, account_id, &vault_delta, &AccountStorageDelta::default()); + let delta = dummy_partial_delta(account_id, vault_delta, AccountStorageDelta::default()); + forest.update_account(block_num, &delta); let vault_root = forest.vault_roots[&(account_id, block_num)]; assert_ne!(vault_root, EMPTY_WORD); } #[test] -fn test_compare_delta_vs_db_vault_with_fungible_asset() { - let account_id = test_account(); - let faucet_id = test_faucet(); +fn test_compare_partial_vs_full_state_delta_vault() { + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); let block_num = BlockNumber::GENESIS.child(); - let asset = create_fungible_asset(faucet_id, 100); + let asset = dummy_fungible_asset(faucet_id, 100); - // Approach 1: Delta-based update - let mut forest_delta = InnerForest::new(); + // Approach 1: Partial delta (simulates block application) + let mut forest_partial = InnerForest::new(); let mut vault_delta = AccountVaultDelta::default(); vault_delta.add_asset(asset).unwrap(); - forest_delta.update_account( - block_num, - account_id, - &vault_delta, - &AccountStorageDelta::default(), - ); + let partial_delta = + dummy_partial_delta(account_id, vault_delta, AccountStorageDelta::default()); + forest_partial.update_account(block_num, &partial_delta); - // Approach 2: DB-based population - let mut forest_db = InnerForest::new(); - let vault_assets = vec![asset]; - forest_db.add_vault(account_id, &vault_assets, block_num); + // Approach 2: Full-state delta (simulates DB reconstruction) + let mut forest_full = InnerForest::new(); + let full_delta = dummy_full_state_delta(account_id, &[asset]); + forest_full.update_account(block_num, &full_delta); - // Both approaches must produce identical roots - let root_delta = forest_delta.vault_roots.get(&(account_id, block_num)).unwrap(); - let root_db = forest_db.vault_roots.get(&(account_id, block_num)).unwrap(); + // Both approaches must produce identical vault roots + let root_partial = forest_partial.vault_roots.get(&(account_id, block_num)).unwrap(); + let root_full = forest_full.vault_roots.get(&(account_id, block_num)).unwrap(); - assert_eq!(root_delta, root_db); - assert_ne!(*root_delta, EMPTY_WORD); + assert_eq!(root_partial, root_full); + assert_ne!(*root_partial, EMPTY_WORD); } #[test] @@ -114,23 +148,57 @@ fn test_slot_names_are_tracked() { #[test] fn test_incremental_vault_updates() { let mut forest = InnerForest::new(); - let account_id = test_account(); - let faucet_id = test_faucet(); - let storage_delta = AccountStorageDelta::default(); + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); // Block 1: 100 tokens let block_1 = BlockNumber::GENESIS.child(); let mut vault_delta_1 = AccountVaultDelta::default(); - vault_delta_1.add_asset(create_fungible_asset(faucet_id, 100)).unwrap(); - forest.update_account(block_1, account_id, &vault_delta_1, &storage_delta); + vault_delta_1.add_asset(dummy_fungible_asset(faucet_id, 100)).unwrap(); + let delta_1 = dummy_partial_delta(account_id, vault_delta_1, AccountStorageDelta::default()); + forest.update_account(block_1, &delta_1); let root_1 = forest.vault_roots[&(account_id, block_1)]; - // Block 2: 150 tokens + // Block 2: 150 tokens (update) let block_2 = block_1.child(); let mut vault_delta_2 = AccountVaultDelta::default(); - vault_delta_2.add_asset(create_fungible_asset(faucet_id, 150)).unwrap(); - forest.update_account(block_2, account_id, &vault_delta_2, &storage_delta); + vault_delta_2.add_asset(dummy_fungible_asset(faucet_id, 150)).unwrap(); + let delta_2 = dummy_partial_delta(account_id, vault_delta_2, AccountStorageDelta::default()); + forest.update_account(block_2, &delta_2); let root_2 = forest.vault_roots[&(account_id, block_2)]; assert_ne!(root_1, root_2); } + +#[test] +fn test_full_state_delta_starts_from_empty_root() { + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); + let block_num = BlockNumber::GENESIS.child(); + + // Simulate a pre-existing vault state that should be ignored for full-state deltas + let mut vault_delta_pre = AccountVaultDelta::default(); + vault_delta_pre.add_asset(dummy_fungible_asset(faucet_id, 999)).unwrap(); + let delta_pre = + dummy_partial_delta(account_id, vault_delta_pre, AccountStorageDelta::default()); + forest.update_account(block_num, &delta_pre); + assert!(forest.vault_roots.contains_key(&(account_id, block_num))); + + // Now create a full-state delta at the same block + // A full-state delta should start from an empty root, not from the previous state + let asset = dummy_fungible_asset(faucet_id, 100); + let full_delta = dummy_full_state_delta(account_id, &[asset]); + + // Create a fresh forest to compare + let mut fresh_forest = InnerForest::new(); + fresh_forest.update_account(block_num, &full_delta); + let fresh_root = fresh_forest.vault_roots[&(account_id, block_num)]; + + // Update the original forest with the full-state delta + forest.update_account(block_num, &full_delta); + let updated_root = forest.vault_roots[&(account_id, block_num)]; + + // The full-state delta should produce the same root regardless of prior state + assert_eq!(updated_root, fresh_root); +} diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 569c5e5de3..4eecb9d346 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -24,7 +24,7 @@ use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; use miden_objects::account::delta::AccountUpdateDetails; -use miden_objects::account::{AccountId, StorageSlotContent, StorageSlotName}; +use miden_objects::account::{AccountId, StorageSlotContent}; use miden_objects::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_objects::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_objects::block::{ @@ -495,19 +495,15 @@ impl State { for (account_id, details) in account_updates { match details { - AccountUpdateDetails::Delta(delta) => { - // Update the forest with vault and storage deltas - forest_guard.update_account( - block_num, - account_id, - delta.vault(), - delta.storage(), - ); + AccountUpdateDetails::Delta(ref delta) => { + // Update the forest with the delta (handles both full-state and partial) + forest_guard.update_account(block_num, delta); tracing::debug!( target: COMPONENT, %account_id, %block_num, + is_full_state = delta.is_full_state(), "Updated forest with account delta" ); }, @@ -526,10 +522,10 @@ impl State { Ok(()) } - /// Updates `SmtForest` from database state (DB-based) + /// Updates `SmtForest` from database state using the unified delta /// - /// This method is used during initial `State::load()` where deltas are not available. - /// For block application, prefer `fn update_forest` which uses deltas directly. + /// Primarily used in `State::load()` where we need to reconstruct + /// the forest from full account state recovered from the database. /// /// # Warning /// @@ -541,25 +537,36 @@ impl State { account_ids: Vec, block_num: BlockNumber, ) -> Result<(), ApplyBlockError> { + use miden_objects::account::delta::AccountDelta; + // Acquire write lock once for the entire initialization let mut forest_guard = self.forest.write().await; - // Process each account, updating both storage maps and vaults + // Process each account for account_id in account_ids { - // Query and update storage maps for this account - let storage = self.db.select_account_storage_at_block(account_id, block_num).await?; - let map_slots = extract_map_slots_from_storage(&storage); - - if !map_slots.is_empty() { - forest_guard.add_storage_map(account_id, map_slots, block_num); + // Skip private accounts - they don't have public state to reconstruct + if !account_id.is_public() { + tracing::trace!( + target: COMPONENT, + %account_id, + %block_num, + "Skipping private account during forest initialization" + ); + continue; } - // Query and update vault for this account - let vault_assets = self.db.select_account_vault_at_block(account_id, block_num).await?; + // Get the full account from the database + let account_info = self.db.select_account(account_id).await?; + let account = account_info + .details + .expect("public accounts always have details in DB"); - if !vault_assets.is_empty() { - forest_guard.add_vault(account_id, &vault_assets, block_num); - } + // Convert the full account to a full-state delta + let delta = AccountDelta::try_from(account) + .expect("accounts from DB should not have seeds"); + + // Use the unified update method (will recognize it's a full-state delta) + forest_guard.update_account(block_num, &delta); tracing::debug!( target: COMPONENT, @@ -1366,31 +1373,3 @@ async fn load_account_tree( Ok(AccountTreeWithHistory::new(account_tree, block_number)) } - -// HELPERS -// ================================================================================================= - -/// Extract storage map slots from a single `AccountStorage` object. -/// -/// # Returns -/// -/// Vector of `(account_id, slot_name, entries)` tuples ready for forest population. -pub(crate) fn extract_map_slots_from_storage( - storage: &miden_objects::account::AccountStorage, -) -> Vec<(StorageSlotName, Vec<(Word, Word)>)> { - use miden_objects::account::StorageSlotContent; - - let mut map_slots = Vec::new(); - - for slot in storage.slots() { - if let StorageSlotContent::Map(map) = slot.content() { - let entries = Vec::from_iter(map.entries().map(|(k, v)| (*k, *v))); - - if !entries.is_empty() { - map_slots.push((slot.name().clone(), entries)); - } - } - } - - map_slots -} From d6b31eff29bd13c099d356cdb9da27791f50d552 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 23 Dec 2025 00:38:40 +0100 Subject: [PATCH 60/95] minor --- crates/store/src/inner_forest.rs | 176 +++++++++++++++---------------- 1 file changed, 86 insertions(+), 90 deletions(-) diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index abdbdd2e98..6dcc19b991 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -34,51 +34,68 @@ impl InnerForest { } } + // HELPERS + // -------------------------------------------------------------------------------------------- + /// Returns the root of an empty SMT. fn empty_smt_root() -> Word { *EmptySubtreeRoots::entry(SMT_DEPTH, 0) } + /// Retrieves the vault SMT root for an account at a given block, defaulting to empty. + fn get_vault_root(&self, account_id: AccountId, block_num: BlockNumber) -> Word { + self.vault_roots + .get(&(account_id, block_num)) + .copied() + .unwrap_or_else(Self::empty_smt_root) + } + + /// Retrieves the storage map SMT root for an account slot at a given block, defaulting to + /// empty. + fn get_storage_root( + &self, + account_id: AccountId, + slot_name: &StorageSlotName, + block_num: BlockNumber, + ) -> Word { + self.storage_roots + .get(&(account_id, slot_name.clone(), block_num)) + .copied() + .unwrap_or_else(Self::empty_smt_root) + } + + // PUBLIC INTERFACE + // -------------------------------------------------------------------------------------------- + /// Updates the forest with account vault and storage changes from a delta. /// - /// This is the unified interface for updating all account state in the forest. - /// It handles both full-state deltas (new accounts or reconstruction from DB) - /// and partial deltas (incremental updates during block application). - /// - /// For full-state deltas (`delta.is_full_state() == true`), the forest is populated - /// from scratch using an empty SMT root. For partial deltas, changes are applied - /// on top of the previous block's state. - /// - /// # Arguments + /// Unified interface for updating all account state in the forest, handling both + /// full-state deltas (new accounts or reconstruction from DB) and partial deltas + /// (incremental updates during block application). /// - /// * `block_num` - Block number for which these changes are being applied - /// * `delta` - The account delta containing vault and storage changes + /// Full-state deltas (`delta.is_full_state() == true`) populate the forest from + /// scratch using an empty SMT root. Partial deltas apply changes on top of the + /// previous block's state. pub(crate) fn update_account(&mut self, block_num: BlockNumber, delta: &AccountDelta) { let account_id = delta.id(); let is_full_state = delta.is_full_state(); - // Update vault if there are any changes if !delta.vault().is_empty() { self.update_account_vault(block_num, account_id, delta.vault(), is_full_state); } - // Update storage maps if there are any changes if !delta.storage().is_empty() { self.update_account_storage(block_num, account_id, delta.storage(), is_full_state); } } + // PRIVATE METHODS + // -------------------------------------------------------------------------------------------- + /// Updates the forest with vault changes from a delta. /// /// Processes both fungible and non-fungible asset changes, building entries /// for the vault SMT and tracking the new root. - /// - /// # Arguments - /// - /// * `block_num` - Block number for this update - /// * `account_id` - The account being updated - /// * `vault_delta` - Changes to the account's asset vault - /// * `is_full_state` - If true, start from empty root; otherwise use previous block's root fn update_account_vault( &mut self, block_num: BlockNumber, @@ -86,73 +103,56 @@ impl InnerForest { vault_delta: &AccountVaultDelta, is_full_state: bool, ) { - // For full-state deltas (new accounts or reconstruction), start from empty root. - // For partial deltas, look up the previous block's root. let prev_root = if is_full_state { Self::empty_smt_root() } else { - let prev_block_num = block_num.parent().unwrap_or_default(); - self.vault_roots - .get(&(account_id, prev_block_num)) - .copied() - .unwrap_or_else(Self::empty_smt_root) + self.get_vault_root(account_id, block_num.parent().unwrap_or_default()) }; - // Collect all vault entry updates let mut entries = Vec::new(); - // Process fungible assets - these require special handling to get current amounts - // Note: We rely on the delta containing the updated amounts, not just the changes + // Process fungible assets for (faucet_id, amount) in vault_delta.fungible().iter() { - let amount_u64 = (*amount).try_into().expect("Amount should be non-negative"); + let amount_u64: u64 = (*amount).try_into().expect("amount is non-negative"); let asset: Asset = FungibleAsset::new(*faucet_id, amount_u64) - .expect("Valid fungible asset from delta") + .expect("valid fungible asset") .into(); entries.push((asset.vault_key().into(), Word::from(asset))); } // Process non-fungible assets for (asset, action) in vault_delta.non_fungible().iter() { - match action { - NonFungibleDeltaAction::Add => { - entries - .push((asset.vault_key().into(), Word::from(Asset::NonFungible(*asset)))); - }, - NonFungibleDeltaAction::Remove => { - entries.push((asset.vault_key().into(), EMPTY_WORD)); - }, - } + let value = match action { + NonFungibleDeltaAction::Add => Word::from(Asset::NonFungible(*asset)), + NonFungibleDeltaAction::Remove => EMPTY_WORD, + }; + entries.push((asset.vault_key().into(), value)); } - if !entries.is_empty() { - let updated_root = self - .storage_forest - .batch_insert(prev_root, entries.iter().copied()) - .expect("Forest insertion should succeed"); + if entries.is_empty() { + return; + } - self.vault_roots.insert((account_id, block_num), updated_root); + let updated_root = self + .storage_forest + .batch_insert(prev_root, entries.iter().copied()) + .expect("forest insertion should succeed"); - tracing::debug!( - target: crate::COMPONENT, - account_id = %account_id, - block_num = %block_num, - vault_entries = entries.len(), - "Updated vault in forest" - ); - } + self.vault_roots.insert((account_id, block_num), updated_root); + + tracing::debug!( + target: crate::COMPONENT, + %account_id, + %block_num, + vault_entries = entries.len(), + "Updated vault in forest" + ); } /// Updates the forest with storage map changes from a delta. /// /// Processes storage map slot deltas, building SMTs for each modified slot /// and tracking the new roots. - /// - /// # Arguments - /// - /// * `block_num` - Block number for this update - /// * `account_id` - The account being updated - /// * `storage_delta` - Changes to the account's storage maps - /// * `is_full_state` - If true, start from empty root; otherwise use previous block's root fn update_account_storage( &mut self, block_num: BlockNumber, @@ -160,44 +160,40 @@ impl InnerForest { storage_delta: &AccountStorageDelta, is_full_state: bool, ) { + let parent_block = block_num.parent().unwrap_or_default(); + for (slot_name, map_delta) in storage_delta.maps() { - // For full-state deltas (new accounts or reconstruction), start from empty root. - // For partial deltas, look up the previous block's root. let prev_root = if is_full_state { Self::empty_smt_root() } else { - let prev_block_num = block_num.parent().unwrap_or_default(); - self.storage_roots - .get(&(account_id, slot_name.clone(), prev_block_num)) - .copied() - .unwrap_or_else(Self::empty_smt_root) + self.get_storage_root(account_id, slot_name, parent_block) }; - // Collect entries from the delta - let entries = map_delta + let entries: Vec<_> = map_delta .entries() .iter() .map(|(key, value)| ((*key).into(), *value)) - .collect::>(); - - if !entries.is_empty() { - let updated_root = self - .storage_forest - .batch_insert(prev_root, entries.iter().copied()) - .expect("Forest insertion should succeed"); - - self.storage_roots - .insert((account_id, slot_name.clone(), block_num), updated_root); - - tracing::debug!( - target: crate::COMPONENT, - account_id = %account_id, - block_num = %block_num, - slot_name = ?slot_name, - entries = entries.len(), - "Updated storage map in forest" - ); + .collect(); + + if entries.is_empty() { + continue; } + + let updated_root = self + .storage_forest + .batch_insert(prev_root, entries.iter().copied()) + .expect("forest insertion should succeed"); + + self.storage_roots.insert((account_id, slot_name.clone(), block_num), updated_root); + + tracing::debug!( + target: crate::COMPONENT, + %account_id, + %block_num, + ?slot_name, + entries = entries.len(), + "Updated storage map in forest" + ); } } } From 9c859fcbc237fcbc4331781115bef4b610105e80 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 23 Dec 2025 02:04:51 +0100 Subject: [PATCH 61/95] fmt --- crates/store/src/inner_forest.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index 6dcc19b991..a8b0d3423f 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -114,9 +114,8 @@ impl InnerForest { // Process fungible assets for (faucet_id, amount) in vault_delta.fungible().iter() { let amount_u64: u64 = (*amount).try_into().expect("amount is non-negative"); - let asset: Asset = FungibleAsset::new(*faucet_id, amount_u64) - .expect("valid fungible asset") - .into(); + let asset: Asset = + FungibleAsset::new(*faucet_id, amount_u64).expect("valid fungible asset").into(); entries.push((asset.vault_key().into(), Word::from(asset))); } @@ -169,11 +168,8 @@ impl InnerForest { self.get_storage_root(account_id, slot_name, parent_block) }; - let entries: Vec<_> = map_delta - .entries() - .iter() - .map(|(key, value)| ((*key).into(), *value)) - .collect(); + let entries: Vec<_> = + map_delta.entries().iter().map(|(key, value)| ((*key).into(), *value)).collect(); if entries.is_empty() { continue; @@ -184,7 +180,8 @@ impl InnerForest { .batch_insert(prev_root, entries.iter().copied()) .expect("forest insertion should succeed"); - self.storage_roots.insert((account_id, slot_name.clone(), block_num), updated_root); + self.storage_roots + .insert((account_id, slot_name.clone(), block_num), updated_root); tracing::debug!( target: crate::COMPONENT, From 579b9dccc3d83aa7bc2d99049e828142ac5cacd4 Mon Sep 17 00:00:00 2001 From: Bobbin Threadbare Date: Sat, 27 Dec 2025 00:13:40 -0800 Subject: [PATCH 62/95] chore: fix merge conflicts --- crates/proto/src/domain/account.rs | 6 +++--- crates/store/src/db/mod.rs | 2 +- crates/store/src/db/models/conv.rs | 2 +- crates/store/src/db/models/queries/accounts.rs | 2 +- crates/store/src/db/tests.rs | 1 + crates/store/src/errors.rs | 5 +++-- crates/store/src/inner_forest.rs | 13 +++++++------ crates/store/src/inner_forest/tests.rs | 12 ++++++------ crates/store/src/state.rs | 14 +++++++++++--- 9 files changed, 34 insertions(+), 23 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 2b354dda79..4c11a44789 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -1,7 +1,6 @@ use std::fmt::{Debug, Display, Formatter}; use miden_node_utils::formatting::format_opt; -use miden_protocol::Word; use miden_protocol::account::{ Account, AccountHeader, @@ -18,6 +17,7 @@ use miden_protocol::block::account_tree::AccountWitness; use miden_protocol::crypto::merkle::SparseMerklePath; use miden_protocol::note::{NoteExecutionMode, NoteTag}; use miden_protocol::utils::{Deserializable, DeserializationError, Serializable}; +use miden_protocol::{AssetError, Word}; use thiserror::Error; use super::try_convert; @@ -100,7 +100,7 @@ impl From<&AccountInfo> for proto::account::AccountDetails { fn from(AccountInfo { summary, details }: &AccountInfo) -> Self { Self { summary: Some(summary.into()), - details: details.as_ref().map(miden_protocol::utils::Serializable::to_bytes), + details: details.as_ref().map(Serializable::to_bytes), } } } @@ -396,7 +396,7 @@ impl AccountVaultDetails { /// rather than extracted from an `AssetVault`. /// /// The entries are `(vault_key, asset)` pairs where `asset` is a Word representation. - pub fn from_entries(entries: Vec<(Word, Word)>) -> Result { + pub fn from_entries(entries: Vec<(Word, Word)>) -> Result { if entries.len() > Self::MAX_RETURN_ENTRIES { return Ok(Self::LimitExceeded); } diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index a6ce167c77..01fda65c93 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -7,7 +7,7 @@ use diesel::{Connection, RunQueryDsl, SqliteConnection}; use miden_node_proto::domain::account::{AccountInfo, AccountSummary, NetworkAccountPrefix}; use miden_node_proto::generated as proto; use miden_protocol::Word; -use miden_protocol::account::AccountId; +use miden_protocol::account::{AccountHeader, AccountId, AccountStorage}; use miden_protocol::asset::{Asset, AssetVaultKey}; use miden_protocol::block::{BlockHeader, BlockNoteIndex, BlockNumber, ProvenBlock}; use miden_protocol::crypto::merkle::SparseMerklePath; diff --git a/crates/store/src/db/models/conv.rs b/crates/store/src/db/models/conv.rs index 73890ded4a..37a9b019fb 100644 --- a/crates/store/src/db/models/conv.rs +++ b/crates/store/src/db/models/conv.rs @@ -34,7 +34,7 @@ use miden_node_proto::domain::account::NetworkAccountPrefix; use miden_protocol::Felt; -use miden_protocol::account::StorageSlotName; +use miden_protocol::account::{StorageSlotName, StorageSlotType}; use miden_protocol::block::BlockNumber; use miden_protocol::note::{NoteExecutionMode, NoteTag}; diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 764f201784..50d5fca3b9 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -37,7 +37,7 @@ use miden_protocol::account::{ use miden_protocol::asset::{Asset, AssetVault, AssetVaultKey, FungibleAsset}; use miden_protocol::block::{BlockAccountUpdate, BlockNumber}; use miden_protocol::utils::{Deserializable, Serializable}; -use miden_protocol::{Felt, Word}; +use miden_protocol::{Felt, FieldElement, Word}; use crate::db::models::conv::{SqlTypeConvert, nonce_to_raw_sql, raw_sql_to_nonce}; use crate::db::models::{serialize_vec, vec_raw_try_into}; diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index d1857067ec..7abd7ad117 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -61,6 +61,7 @@ use miden_protocol::transaction::{ TransactionHeader, TransactionId, }; +use miden_protocol::utils::Serializable; use miden_protocol::{EMPTY_WORD, Felt, FieldElement, Word, ZERO}; use miden_standards::account::auth::AuthRpoFalcon512; use miden_standards::code_builder::CodeBuilder; diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index d88d757774..7471c0b583 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -8,9 +8,10 @@ use miden_node_proto::errors::{ConversionError, GrpcError}; use miden_node_utils::limiter::QueryLimitError; use miden_protocol::account::AccountId; use miden_protocol::block::BlockNumber; +use miden_protocol::crypto::merkle::MerkleError; use miden_protocol::crypto::merkle::mmr::MmrError; use miden_protocol::crypto::utils::DeserializationError; -use miden_protocol::note::Nullifier; +use miden_protocol::note::{NoteId, Nullifier}; use miden_protocol::transaction::OutputNote; use miden_protocol::{ AccountDeltaError, @@ -56,7 +57,7 @@ pub enum DatabaseError { #[error("I/O error")] IoError(#[from] io::Error), #[error("merkle error")] - MerkleError(#[from] miden_protocol::crypto::merkle::MerkleError), + MerkleError(#[from] MerkleError), #[error("network account error")] NetworkAccountError(#[from] NetworkAccountError), #[error("note error")] diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index a8b0d3423f..81e6f8d512 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -1,11 +1,12 @@ use std::collections::BTreeMap; -use miden_objects::account::delta::{AccountDelta, AccountStorageDelta, AccountVaultDelta}; -use miden_objects::account::{AccountId, NonFungibleDeltaAction, StorageSlotName}; -use miden_objects::asset::{Asset, FungibleAsset}; -use miden_objects::block::BlockNumber; -use miden_objects::crypto::merkle::{EmptySubtreeRoots, SMT_DEPTH, SmtForest}; -use miden_objects::{EMPTY_WORD, Word}; +use miden_protocol::account::delta::{AccountDelta, AccountStorageDelta, AccountVaultDelta}; +use miden_protocol::account::{AccountId, NonFungibleDeltaAction, StorageSlotName}; +use miden_protocol::asset::{Asset, FungibleAsset}; +use miden_protocol::block::BlockNumber; +use miden_protocol::crypto::merkle::EmptySubtreeRoots; +use miden_protocol::crypto::merkle::smt::{SMT_DEPTH, SmtForest}; +use miden_protocol::{EMPTY_WORD, Word}; #[cfg(test)] mod tests; diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs index 868d59871b..046072a72e 100644 --- a/crates/store/src/inner_forest/tests.rs +++ b/crates/store/src/inner_forest/tests.rs @@ -1,10 +1,10 @@ -use miden_objects::account::AccountCode; -use miden_objects::asset::{Asset, AssetVault, FungibleAsset}; -use miden_objects::testing::account_id::{ +use miden_protocol::account::AccountCode; +use miden_protocol::asset::{Asset, AssetVault, FungibleAsset}; +use miden_protocol::testing::account_id::{ ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET, ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE, }; -use miden_objects::{Felt, FieldElement}; +use miden_protocol::{Felt, FieldElement}; use super::*; @@ -37,7 +37,7 @@ fn dummy_partial_delta( /// Creates a full-state `AccountDelta` (with code) for testing DB reconstruction. fn dummy_full_state_delta(account_id: AccountId, assets: &[Asset]) -> AccountDelta { - use miden_objects::account::{Account, AccountStorage}; + use miden_protocol::account::{Account, AccountStorage}; // Create a minimal account with the given assets let vault = AssetVault::new(assets).unwrap(); @@ -53,7 +53,7 @@ fn dummy_full_state_delta(account_id: AccountId, assets: &[Asset]) -> AccountDel #[test] fn test_empty_smt_root_is_recognized() { - use miden_objects::crypto::merkle::Smt; + use miden_protocol::crypto::merkle::smt::Smt; let empty_root = InnerForest::empty_smt_root(); diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 780ec20d2a..c8460d7c18 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -23,10 +23,18 @@ use miden_node_proto::domain::account::{ use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; -use miden_protocol::account::{AccountHeader, AccountId, StorageSlot, StorageSlotContent}; +use miden_protocol::account::delta::AccountUpdateDetails; +use miden_protocol::account::{AccountId, StorageSlotContent}; use miden_protocol::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; -use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, ProvenBlock}; +use miden_protocol::block::{ + BlockHeader, + BlockInputs, + BlockNoteTree, + BlockNumber, + Blockchain, + ProvenBlock, +}; use miden_protocol::crypto::merkle::mmr::{Forest, Mmr, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; use miden_protocol::crypto::merkle::smt::{ LargeSmt, @@ -524,7 +532,7 @@ impl State { account_ids: Vec, block_num: BlockNumber, ) -> Result<(), ApplyBlockError> { - use miden_objects::account::delta::AccountDelta; + use miden_protocol::account::delta::AccountDelta; // Acquire write lock once for the entire initialization let mut forest_guard = self.forest.write().await; From 3336edbaec8a8312302147678eb4058ddfeeb50a Mon Sep 17 00:00:00 2001 From: Bobbin Threadbare Date: Sat, 27 Dec 2025 00:22:19 -0800 Subject: [PATCH 63/95] chore: fix test --- crates/store/src/db/tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 7abd7ad117..030f3a6a9c 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1516,7 +1516,7 @@ fn test_select_account_code_at_block_with_updates() { // Create initial account with code v1 at block 1 let code_v1_str = "\ - export.account_procedure_1 + pub proc account_procedure_1 push.1.2 add end @@ -1539,7 +1539,7 @@ fn test_select_account_code_at_block_with_updates() { // Create account with different code v2 at block 2 let code_v2_str = "\ - export.account_procedure_1 + pub proc account_procedure_1 push.3.4 mul end @@ -1567,7 +1567,7 @@ fn test_select_account_code_at_block_with_updates() { // Create account with different code v3 at block 3 let code_v3_str = "\ - export.account_procedure_1 + pub proc account_procedure_1 push.5.6 sub end From 2aa8c8b3d5c21ef840c7218a904cb57631c5fd0d Mon Sep 17 00:00:00 2001 From: Bobbin Threadbare Date: Sat, 27 Dec 2025 00:40:09 -0800 Subject: [PATCH 64/95] chore: minor formatting changes --- crates/store/src/inner_forest.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest.rs index 81e6f8d512..197d9e8727 100644 --- a/crates/store/src/inner_forest.rs +++ b/crates/store/src/inner_forest.rs @@ -11,11 +11,14 @@ use miden_protocol::{EMPTY_WORD, Word}; #[cfg(test)] mod tests; +// INNER FOREST +// ================================================================================================ + /// Container for forest-related state that needs to be updated atomically. pub(crate) struct InnerForest { /// `SmtForest` for efficient account storage reconstruction. /// Populated during block import with storage and vault SMTs. - pub(crate) storage_forest: SmtForest, + forest: SmtForest, /// Maps (`account_id`, `slot_name`, `block_num`) to SMT root. /// Populated during block import for all storage map slots. @@ -29,7 +32,7 @@ pub(crate) struct InnerForest { impl InnerForest { pub(crate) fn new() -> Self { Self { - storage_forest: SmtForest::new(), + forest: SmtForest::new(), storage_roots: BTreeMap::new(), vault_roots: BTreeMap::new(), } @@ -39,7 +42,7 @@ impl InnerForest { // -------------------------------------------------------------------------------------------- /// Returns the root of an empty SMT. - fn empty_smt_root() -> Word { + const fn empty_smt_root() -> Word { *EmptySubtreeRoots::entry(SMT_DEPTH, 0) } @@ -70,13 +73,12 @@ impl InnerForest { /// Updates the forest with account vault and storage changes from a delta. /// - /// Unified interface for updating all account state in the forest, handling both - /// full-state deltas (new accounts or reconstruction from DB) and partial deltas - /// (incremental updates during block application). + /// Unified interface for updating all account state in the forest, handling both full-state + /// deltas (new accounts or reconstruction from DB) and partial deltas (incremental updates + /// during block application). /// - /// Full-state deltas (`delta.is_full_state() == true`) populate the forest from - /// scratch using an empty SMT root. Partial deltas apply changes on top of the - /// previous block's state. + /// Full-state deltas (`delta.is_full_state() == true`) populate the forest from scratch using + /// an empty SMT root. Partial deltas apply changes on top of the previous block's state. pub(crate) fn update_account(&mut self, block_num: BlockNumber, delta: &AccountDelta) { let account_id = delta.id(); let is_full_state = delta.is_full_state(); @@ -95,8 +97,8 @@ impl InnerForest { /// Updates the forest with vault changes from a delta. /// - /// Processes both fungible and non-fungible asset changes, building entries - /// for the vault SMT and tracking the new root. + /// Processes both fungible and non-fungible asset changes, building entries for the vault SMT + /// and tracking the new root. fn update_account_vault( &mut self, block_num: BlockNumber, @@ -134,7 +136,7 @@ impl InnerForest { } let updated_root = self - .storage_forest + .forest .batch_insert(prev_root, entries.iter().copied()) .expect("forest insertion should succeed"); @@ -177,7 +179,7 @@ impl InnerForest { } let updated_root = self - .storage_forest + .forest .batch_insert(prev_root, entries.iter().copied()) .expect("forest insertion should succeed"); From 354d5864d661f9fb9c007195f5f29b1552ac9b3b Mon Sep 17 00:00:00 2001 From: Bobbin Threadbare Date: Sat, 27 Dec 2025 01:22:36 -0800 Subject: [PATCH 65/95] chore: move InnerForest module --- .../{inner_forest.rs => inner_forest/mod.rs} | 0 crates/store/src/state.rs | 128 ++++++++++-------- 2 files changed, 73 insertions(+), 55 deletions(-) rename crates/store/src/{inner_forest.rs => inner_forest/mod.rs} (100%) diff --git a/crates/store/src/inner_forest.rs b/crates/store/src/inner_forest/mod.rs similarity index 100% rename from crates/store/src/inner_forest.rs rename to crates/store/src/inner_forest/mod.rs diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index c8460d7c18..cbac397762 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -109,7 +109,16 @@ where } } -/// The rollup state +// CHAIN STATE +// ================================================================================================ + +/// The chain state. +/// +/// The chain state consists of three main components: +/// - A persistent database that stores notes, nullifiers, recent account states, and related data. +/// - In-memory data structures contain Merkle paths for various objects - e.g., all accounts, +/// nullifiers, public account vaults and storage, MMR of all block headers. +/// - Raw block data for all blocks that is stored on disk as flat files. pub struct State { /// The database which stores block headers, nullifiers, notes, and the latest states of /// accounts. @@ -132,6 +141,9 @@ pub struct State { } impl State { + // CONSTRUCTOR + // -------------------------------------------------------------------------------------------- + /// Loads the state from the `db`. #[instrument(target = COMPONENT, skip_all)] pub async fn load(data_path: &Path) -> Result { @@ -184,6 +196,64 @@ impl State { Ok(me) } + /// Updates `SmtForest` from database state using the unified delta. + /// + /// Primarily used in `State::load()` where we need to reconstruct the forest from full account + /// state recovered from the database. + /// + /// # Warning + /// + /// Has internal locking to mutate the state, use cautiously in scopes with other mutex guards + /// around! + #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num))] + async fn initialize_storage_forest_from_db( + &self, + account_ids: Vec, + block_num: BlockNumber, + ) -> Result<(), ApplyBlockError> { + use miden_protocol::account::delta::AccountDelta; + + // Acquire write lock once for the entire initialization + let mut forest_guard = self.forest.write().await; + + // Process each account + for account_id in account_ids { + // Skip private accounts - they don't have public state to reconstruct + if !account_id.is_public() { + tracing::trace!( + target: COMPONENT, + %account_id, + %block_num, + "Skipping private account during forest initialization" + ); + continue; + } + + // Get the full account from the database + let account_info = self.db.select_account(account_id).await?; + let account = account_info.details.expect("public accounts always have details in DB"); + + // Convert the full account to a full-state delta + let delta = + AccountDelta::try_from(account).expect("accounts from DB should not have seeds"); + + // Use the unified update method (will recognize it's a full-state delta) + forest_guard.update_account(block_num, &delta); + + tracing::debug!( + target: COMPONENT, + %account_id, + %block_num, + "Initialized forest for account from DB" + ); + } + + Ok(()) + } + + // STATE MUTATOR + // -------------------------------------------------------------------------------------------- + /// Apply changes of a new block to the DB and in-memory data structures. /// /// ## Note on state consistency @@ -517,60 +587,8 @@ impl State { Ok(()) } - /// Updates `SmtForest` from database state using the unified delta - /// - /// Primarily used in `State::load()` where we need to reconstruct - /// the forest from full account state recovered from the database. - /// - /// # Warning - /// - /// Has internal locking to mutate the state, use cautiously in scopes with other - /// mutex guards around! - #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num))] - async fn initialize_storage_forest_from_db( - &self, - account_ids: Vec, - block_num: BlockNumber, - ) -> Result<(), ApplyBlockError> { - use miden_protocol::account::delta::AccountDelta; - - // Acquire write lock once for the entire initialization - let mut forest_guard = self.forest.write().await; - - // Process each account - for account_id in account_ids { - // Skip private accounts - they don't have public state to reconstruct - if !account_id.is_public() { - tracing::trace!( - target: COMPONENT, - %account_id, - %block_num, - "Skipping private account during forest initialization" - ); - continue; - } - - // Get the full account from the database - let account_info = self.db.select_account(account_id).await?; - let account = account_info.details.expect("public accounts always have details in DB"); - - // Convert the full account to a full-state delta - let delta = - AccountDelta::try_from(account).expect("accounts from DB should not have seeds"); - - // Use the unified update method (will recognize it's a full-state delta) - forest_guard.update_account(block_num, &delta); - - tracing::debug!( - target: COMPONENT, - %account_id, - %block_num, - "Initialized forest for account from DB" - ); - } - - Ok(()) - } + // STATE ACCESSORS + // -------------------------------------------------------------------------------------------- /// Queries a [BlockHeader] from the database, and returns it alongside its inclusion proof. /// From a96def07bfcabc8b80a244d911e85824e05fa964 Mon Sep 17 00:00:00 2001 From: Bobbin Threadbare Date: Sat, 27 Dec 2025 09:36:59 -0800 Subject: [PATCH 66/95] chore: refactor SMT forest initialization --- crates/store/src/state.rs | 163 ++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 96 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index cbac397762..bfc526bc57 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -35,7 +35,7 @@ use miden_protocol::block::{ Blockchain, ProvenBlock, }; -use miden_protocol::crypto::merkle::mmr::{Forest, Mmr, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; +use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; use miden_protocol::crypto::merkle::smt::{ LargeSmt, LargeSmtError, @@ -160,95 +160,19 @@ impl State { .await .map_err(StateInitializationError::DatabaseLoadError)?; - let chain_mmr = load_mmr(&mut db).await?; - let block_headers = db.select_all_block_headers().await?; - let latest_block_num = - block_headers.last().map_or(BlockNumber::GENESIS, BlockHeader::block_num); + let blockchain = load_mmr(&mut db).await?; + let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); let account_tree = load_account_tree(&mut db, latest_block_num).await?; let nullifier_tree = load_nullifier_tree(&mut db).await?; + let forest = load_smt_forest(&mut db, latest_block_num).await?; - let inner = RwLock::new(InnerState { - nullifier_tree, - // SAFETY: We assume the loaded MMR is valid and does not have more than u32::MAX - // entries. - blockchain: Blockchain::from_mmr_unchecked(chain_mmr), - account_tree, - }); + let inner = RwLock::new(InnerState { nullifier_tree, blockchain, account_tree }); - let forest = RwLock::new(InnerForest::new()); + let forest = RwLock::new(forest); let writer = Mutex::new(()); let db = Arc::new(db); - let me = Self { db, block_store, inner, forest, writer }; - - // load all accounts from the table - let acc_account_ids = me.db.select_all_account_commitments().await?; - let acc_account_ids = - Vec::from_iter(acc_account_ids.into_iter().map(|(account_id, _)| account_id)); - me.initialize_storage_forest_from_db(acc_account_ids, latest_block_num) - .await - .map_err(|e| { - StateInitializationError::DatabaseError(DatabaseError::InteractError(format!( - "Failed to update storage forest: {e}" - ))) - })?; - - Ok(me) - } - - /// Updates `SmtForest` from database state using the unified delta. - /// - /// Primarily used in `State::load()` where we need to reconstruct the forest from full account - /// state recovered from the database. - /// - /// # Warning - /// - /// Has internal locking to mutate the state, use cautiously in scopes with other mutex guards - /// around! - #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num))] - async fn initialize_storage_forest_from_db( - &self, - account_ids: Vec, - block_num: BlockNumber, - ) -> Result<(), ApplyBlockError> { - use miden_protocol::account::delta::AccountDelta; - - // Acquire write lock once for the entire initialization - let mut forest_guard = self.forest.write().await; - - // Process each account - for account_id in account_ids { - // Skip private accounts - they don't have public state to reconstruct - if !account_id.is_public() { - tracing::trace!( - target: COMPONENT, - %account_id, - %block_num, - "Skipping private account during forest initialization" - ); - continue; - } - - // Get the full account from the database - let account_info = self.db.select_account(account_id).await?; - let account = account_info.details.expect("public accounts always have details in DB"); - - // Convert the full account to a full-state delta - let delta = - AccountDelta::try_from(account).expect("accounts from DB should not have seeds"); - - // Use the unified update method (will recognize it's a full-state delta) - forest_guard.update_account(block_num, &delta); - - tracing::debug!( - target: COMPONENT, - %account_id, - %block_num, - "Initialized forest for account from DB" - ); - } - - Ok(()) + Ok(Self { db, block_store, inner, forest, writer }) } // STATE MUTATOR @@ -1327,9 +1251,25 @@ impl State { } } -// UTILITIES +// INNER STATE LOADING // ================================================================================================ +#[instrument(level = "info", target = COMPONENT, skip_all)] +async fn load_mmr(db: &mut Db) -> Result { + let block_commitments: Vec = db + .select_all_block_headers() + .await? + .iter() + .map(BlockHeader::commitment) + .collect(); + + // SAFETY: We assume the loaded MMR is valid and does not have more than u32::MAX + // entries. + let chain_mmr = Blockchain::from_mmr_unchecked(block_commitments.into()); + + Ok(chain_mmr) +} + #[instrument(level = "info", target = COMPONENT, skip_all)] async fn load_nullifier_tree( db: &mut Db, @@ -1344,18 +1284,6 @@ async fn load_nullifier_tree( .map_err(StateInitializationError::FailedToCreateNullifierTree) } -#[instrument(level = "info", target = COMPONENT, skip_all)] -async fn load_mmr(db: &mut Db) -> Result { - let block_commitments: Vec = db - .select_all_block_headers() - .await? - .iter() - .map(BlockHeader::commitment) - .collect(); - - Ok(block_commitments.into()) -} - #[instrument(level = "info", target = COMPONENT, skip_all)] async fn load_account_tree( db: &mut Db, @@ -1384,3 +1312,46 @@ async fn load_account_tree( Ok(AccountTreeWithHistory::new(account_tree, block_number)) } + +/// Loads SMT forest with storage map and vault Merkle paths for all public accounts. +#[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num))] +async fn load_smt_forest( + db: &mut Db, + block_num: BlockNumber, +) -> Result { + use miden_protocol::account::delta::AccountDelta; + + // Skip private accounts - they don't have public state to reconstruct + let public_account_ids: Vec = db + .select_all_account_commitments() + .await? + .iter() + .filter_map(|(id, _commitment)| if id.has_public_state() { Some(*id) } else { None }) + .collect(); + + // Acquire write lock once for the entire initialization + let mut forest = InnerForest::new(); + + // Process each account + for account_id in public_account_ids { + // Get the full account from the database + let account_info = db.select_account(account_id).await?; + let account = account_info.details.expect("public accounts always have details in DB"); + + // Convert the full account to a full-state delta + let delta = + AccountDelta::try_from(account).expect("accounts from DB should not have seeds"); + + // Use the unified update method (will recognize it's a full-state delta) + forest.update_account(block_num, &delta); + + tracing::debug!( + target: COMPONENT, + %account_id, + %block_num, + "Initialized forest for account from DB" + ); + } + + Ok(forest) +} From e8cdad1b8632e17f4426fca85716b3a599385e6f Mon Sep 17 00:00:00 2001 From: Bobbin Threadbare Date: Sat, 27 Dec 2025 12:30:27 -0800 Subject: [PATCH 67/95] chore: re-organize account queries --- .../store/src/db/models/queries/accounts.rs | 368 ++++-------------- .../db/models/queries/accounts/at_block.rs | 252 ++++++++++++ 2 files changed, 332 insertions(+), 288 deletions(-) create mode 100644 crates/store/src/db/models/queries/accounts/at_block.rs diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 50d5fca3b9..72b1bce745 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -22,12 +22,12 @@ use miden_node_utils::limiter::{ QueryParamAccountIdLimit, QueryParamLimiter, }; +use miden_protocol::Word; use miden_protocol::account::delta::AccountUpdateDetails; use miden_protocol::account::{ Account, AccountCode, AccountDelta, - AccountHeader, AccountId, AccountStorage, NonFungibleDeltaAction, @@ -37,15 +37,25 @@ use miden_protocol::account::{ use miden_protocol::asset::{Asset, AssetVault, AssetVaultKey, FungibleAsset}; use miden_protocol::block::{BlockAccountUpdate, BlockNumber}; use miden_protocol::utils::{Deserializable, Serializable}; -use miden_protocol::{Felt, FieldElement, Word}; use crate::db::models::conv::{SqlTypeConvert, nonce_to_raw_sql, raw_sql_to_nonce}; use crate::db::models::{serialize_vec, vec_raw_try_into}; use crate::db::{AccountVaultValue, schema}; use crate::errors::DatabaseError; +mod at_block; +pub(crate) use at_block::{ + select_account_code_at_block, + select_account_header_at_block, + select_account_storage_at_block, + select_account_vault_at_block, +}; + type StorageMapValueRow = (i64, String, Vec, Vec); +// ACCOUNT RETRIEVAL +// ================================================================================================ + /// Select account by ID from the DB using the given [`SqliteConnection`]. /// /// # Returns @@ -81,7 +91,7 @@ pub(crate) fn select_account( // Backfill account details from database // For private accounts, we don't store full details in the database let details = if account_id.is_public() { - Some(reconstruct_full_account_from_db(conn, account_id)?) + Some(select_full_account(conn, account_id)?) } else { None }; @@ -89,6 +99,67 @@ pub(crate) fn select_account( Ok(AccountInfo { summary, details }) } +/// Reconstruct full Account from database tables for the latest account state +/// +/// This function queries the database tables to reconstruct a complete Account object: +/// - Code from `account_codes` table +/// - Nonce and storage header from `accounts` table +/// - Storage map entries from `account_storage_map_values` table +/// - Vault from `account_vault_assets` table +/// +/// # Note +/// +/// A stop-gap solution to retain store API and construct `AccountInfo` types. +/// The function should ultimately be removed, and any queries be served from the +/// `State` which contains an `SmtForest` to serve the latest and most recent +/// historical data. +// TODO: remove eventually once refactoring is complete +fn select_full_account( + conn: &mut SqliteConnection, + account_id: AccountId, +) -> Result { + // Get account metadata (nonce, code_commitment) and code in a single join query + let (nonce, code_bytes): (Option, Vec) = SelectDsl::select( + schema::accounts::table.inner_join(schema::account_codes::table), + (schema::accounts::nonce, schema::account_codes::code), + ) + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .get_result(conn) + .optional()? + .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + + let nonce = raw_sql_to_nonce(nonce.ok_or_else(|| { + DatabaseError::DataCorrupted(format!("No nonce found for account {account_id}")) + })?); + + let code = AccountCode::read_from_bytes(&code_bytes)?; + + // Reconstruct storage using existing helper function + let storage = select_latest_account_storage(conn, account_id)?; + + // Reconstruct vault from account_vault_assets table + let vault_entries: Vec<(Vec, Option>)> = SelectDsl::select( + schema::account_vault_assets::table, + (schema::account_vault_assets::vault_key, schema::account_vault_assets::asset), + ) + .filter(schema::account_vault_assets::account_id.eq(account_id.to_bytes())) + .filter(schema::account_vault_assets::is_latest.eq(true)) + .load(conn)?; + + let mut assets = Vec::new(); + for (_key_bytes, maybe_asset_bytes) in vault_entries { + if let Some(asset_bytes) = maybe_asset_bytes { + let asset = Asset::read_from_bytes(&asset_bytes)?; + assets.push(asset); + } + } + + let vault = AssetVault::new(&assets)?; + + Ok(Account::new(account_id, vault, storage, code, nonce, None)?) +} + /// Select the latest account info by account ID prefix from the DB using the given /// [`SqliteConnection`]. Meant to be used by the network transaction builder. /// Because network notes get matched through accounts through the account's 30-bit prefix, it is @@ -129,7 +200,7 @@ pub(crate) fn select_account_by_id_prefix( let summary: AccountSummary = raw.try_into()?; let account_id = summary.account_id; // Backfill account details from database - let details = reconstruct_full_account_from_db(conn, account_id).ok(); + let details = select_full_account(conn, account_id).ok(); Ok(Some(AccountInfo { summary, details })) }, } @@ -335,7 +406,7 @@ pub(crate) fn select_all_accounts( // Backfill account details from database let account_infos = Vec::from_iter(summaries.into_iter().map(|summary| { let account_id = summary.account_id; - let details = reconstruct_full_account_from_db(conn, account_id).ok(); + let details = select_full_account(conn, account_id).ok(); AccountInfo { summary, details } })); @@ -497,38 +568,6 @@ pub(crate) fn select_account_storage_map_values( Ok(StorageMapValuesPage { last_block_included, values }) } -/// Returns account storage header at a given block by reading from `accounts.storage_header` -/// and deserializing the storage header blob. -pub(crate) fn select_account_storage_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result { - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - - // Query storage blob for this account at this block - let storage_blob: Option> = - SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) - .filter(schema::accounts::account_id.eq(&account_id_bytes)) - .filter(schema::accounts::block_num.le(block_num_sql)) - .order(schema::accounts::block_num.desc()) - .limit(1) - .first(conn) - .optional()? - .flatten(); - - let Some(blob) = storage_blob else { - // No storage means empty storage - return Ok(AccountStorage::new(Vec::new())?); - }; - - // Deserialize the full AccountStorage from the blob - let storage = AccountStorage::read_from_bytes(&blob)?; - - Ok(storage) -} - /// Select latest account storage header by querying `accounts.storage_header` where /// `is_latest=true`. pub(crate) fn select_latest_account_storage( @@ -557,6 +596,9 @@ pub(crate) fn select_latest_account_storage( Ok(storage) } +// ACCOUNT MUTATION +// ================================================================================================ + #[derive(Queryable, Selectable)] #[diesel(table_name = crate::db::schema::account_vault_assets)] #[diesel(check_for_backend(diesel::sqlite::Sqlite))] @@ -690,67 +732,6 @@ pub(crate) fn insert_account_storage_map_value( Ok(update_count + insert_count) } -/// Reconstruct full Account from database tables for the latest account state -/// -/// This function queries the database tables to reconstruct a complete Account object: -/// - Code from `account_codes` table -/// - Nonce and storage header from `accounts` table -/// - Storage map entries from `account_storage_map_values` table -/// - Vault from `account_vault_assets` table -/// -/// # Note -/// -/// A stop-gap solution to retain store API and construct `AccountInfo` types. -/// The function should ultimately be removed, and any queries be served from the -/// `State` which contains an `SmtForest` to serve the latest and most recent -/// historical data. -// TODO: remove eventually once refactoring is complete -fn reconstruct_full_account_from_db( - conn: &mut SqliteConnection, - account_id: AccountId, -) -> Result { - // Get account metadata (nonce, code_commitment) and code in a single join query - let (nonce, code_bytes): (Option, Vec) = SelectDsl::select( - schema::accounts::table.inner_join(schema::account_codes::table), - (schema::accounts::nonce, schema::account_codes::code), - ) - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::is_latest.eq(true)) - .get_result(conn) - .optional()? - .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; - - let nonce = raw_sql_to_nonce(nonce.ok_or_else(|| { - DatabaseError::DataCorrupted(format!("No nonce found for account {account_id}")) - })?); - - let code = AccountCode::read_from_bytes(&code_bytes)?; - - // Reconstruct storage using existing helper function - let storage = select_latest_account_storage(conn, account_id)?; - - // Reconstruct vault from account_vault_assets table - let vault_entries: Vec<(Vec, Option>)> = SelectDsl::select( - schema::account_vault_assets::table, - (schema::account_vault_assets::vault_key, schema::account_vault_assets::asset), - ) - .filter(schema::account_vault_assets::account_id.eq(account_id.to_bytes())) - .filter(schema::account_vault_assets::is_latest.eq(true)) - .load(conn)?; - - let mut assets = Vec::new(); - for (_key_bytes, maybe_asset_bytes) in vault_entries { - if let Some(asset_bytes) = maybe_asset_bytes { - let asset = Asset::read_from_bytes(&asset_bytes)?; - assets.push(asset); - } - } - - let vault = AssetVault::new(&assets)?; - - Ok(Account::new(account_id, vault, storage, code, nonce, None)?) -} - /// Attention: Assumes the account details are NOT null! The schema explicitly allows this though! #[allow(clippy::too_many_lines)] pub(crate) fn upsert_accounts( @@ -817,7 +798,7 @@ pub(crate) fn upsert_accounts( AccountUpdateDetails::Delta(delta) => { // Reconstruct the full account from database tables - let account = reconstruct_full_account_from_db(conn, account_id)?; + let account = select_full_account(conn, account_id)?; // --- collect storage map updates ---------------------------- @@ -996,192 +977,3 @@ pub(crate) struct AccountStorageMapRowInsert { pub(crate) value: Vec, pub(crate) is_latest: bool, } - -/// Query vault assets at a specific block by finding the most recent update for each `vault_key`. -pub(crate) fn select_account_vault_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result, DatabaseError> { - use schema::account_vault_assets as t; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - - // Since Diesel doesn't support composite keys in subqueries easily, we use a two-step approach: - // Step 1: Get max block_num for each vault_key - let latest_blocks_per_vault_key = Vec::from_iter( - QueryDsl::select( - t::table - .filter(t::account_id.eq(&account_id_bytes)) - .filter(t::block_num.le(block_num_sql)) - .group_by(t::vault_key), - (t::vault_key, diesel::dsl::max(t::block_num)), - ) - .load::<(Vec, Option)>(conn)? - .into_iter() - .filter_map(|(key, maybe_block)| maybe_block.map(|block| (key, block))), - ); - - if latest_blocks_per_vault_key.is_empty() { - return Ok(Vec::new()); - } - - // Step 2: Fetch the full rows matching (vault_key, block_num) pairs - let mut assets = Vec::new(); - for (vault_key_bytes, max_block) in latest_blocks_per_vault_key { - let result: Option>> = QueryDsl::select( - t::table.filter( - t::account_id - .eq(&account_id_bytes) - .and(t::vault_key.eq(&vault_key_bytes)) - .and(t::block_num.eq(max_block)), - ), - t::asset, - ) - .first(conn) - .optional()?; - if let Some(Some(asset_bytes)) = result { - let asset = Asset::read_from_bytes(&asset_bytes)?; - assets.push(asset); - } - } - - // Sort by vault_key for consistent ordering - assets.sort_by_key(Asset::vault_key); - - Ok(assets) -} - -/// Queries the account code for a specific account at a specific block number. -/// -/// Returns `None` if: -/// - The account doesn't exist at that block -/// - The account has no code (private account or account without code commitment) -/// -/// # Arguments -/// -/// * `conn` - Database connection -/// * `account_id` - The account ID to query -/// * `block_num` - The block number at which to query the account code -/// -/// # Returns -/// -/// * `Ok(Some(Vec))` - The account code bytes if found -/// * `Ok(None)` - If account doesn't exist or has no code -/// * `Err(DatabaseError)` - If there's a database error -pub(crate) fn select_account_code_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result>, DatabaseError> { - use schema::{account_codes, accounts}; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = i64::from(block_num.as_u32()); - // Query the accounts table to get the code_commitment at the specified block or earlier - // Then join with account_codes to get the actual code - let result: Option> = SelectDsl::select( - accounts::table - .inner_join(account_codes::table) - .filter(accounts::account_id.eq(&account_id_bytes)) - .filter(accounts::block_num.le(block_num_sql)) - .order(accounts::block_num.desc()) - .limit(1), - account_codes::code, - ) - .first(conn) - .optional()?; - - Ok(result) -} - -#[derive(Debug, Clone, Queryable)] -struct AccountHeaderDataRaw { - code_commitment: Option>, - nonce: Option, - storage_header: Option>, -} - -/// Queries the account header for a specific account at a specific block number. -/// -/// This reconstructs the `AccountHeader` by reading from the `accounts` table: -/// - `account_id`, `nonce`, `code_commitment`, `storage_header`, `vault_root` -/// -/// Returns `None` if the account doesn't exist at that block. -/// -/// # Arguments -/// -/// * `conn` - Database connection -/// * `account_id` - The account ID to query -/// * `block_num` - The block number at which to query the account header -/// -/// # Returns -/// -/// * `Ok(Some(AccountHeader))` - The account header if found -/// * `Ok(None)` - If account doesn't exist at that block -/// * `Err(DatabaseError)` - If there's a database error -pub(crate) fn select_account_header_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result, DatabaseError> { - use schema::accounts; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - - let account_data: Option<(AccountHeaderDataRaw, Option>)> = SelectDsl::select( - accounts::table - .filter(accounts::account_id.eq(&account_id_bytes)) - .filter(accounts::block_num.le(block_num_sql)) - .order(accounts::block_num.desc()) - .limit(1), - ( - (accounts::code_commitment, accounts::nonce, accounts::storage_header), - accounts::vault_root, - ), - ) - .first(conn) - .optional()?; - - let Some(( - AccountHeaderDataRaw { - code_commitment: code_commitment_bytes, - nonce: nonce_raw, - storage_header: storage_header_blob, - }, - vault_root_bytes, - )) = account_data - else { - return Ok(None); - }; - - let storage_commitment = match storage_header_blob { - Some(blob) => { - let storage = AccountStorage::read_from_bytes(&blob)?; - storage.to_commitment() - }, - None => Word::default(), - }; - - let code_commitment = code_commitment_bytes - .map(|bytes| Word::read_from_bytes(&bytes)) - .transpose()? - .unwrap_or(Word::default()); - - let nonce = nonce_raw.map_or(Felt::ZERO, raw_sql_to_nonce); - - let vault_root = vault_root_bytes - .map(|bytes| Word::read_from_bytes(&bytes)) - .transpose()? - .unwrap_or(Word::default()); - - Ok(Some(AccountHeader::new( - account_id, - nonce, - vault_root, - storage_commitment, - code_commitment, - ))) -} diff --git a/crates/store/src/db/models/queries/accounts/at_block.rs b/crates/store/src/db/models/queries/accounts/at_block.rs new file mode 100644 index 0000000000..6ebad2531d --- /dev/null +++ b/crates/store/src/db/models/queries/accounts/at_block.rs @@ -0,0 +1,252 @@ +use diesel::prelude::Queryable; +use diesel::query_dsl::methods::SelectDsl; +use diesel::{ + BoolExpressionMethods, + ExpressionMethods, + OptionalExtension, + QueryDsl, + RunQueryDsl, + SqliteConnection, +}; +use miden_protocol::account::{AccountHeader, AccountId, AccountStorage}; +use miden_protocol::asset::Asset; +use miden_protocol::block::BlockNumber; +use miden_protocol::utils::{Deserializable, Serializable}; +use miden_protocol::{Felt, FieldElement, Word}; + +use crate::db::models::conv::{SqlTypeConvert, raw_sql_to_nonce}; +use crate::db::schema; +use crate::errors::DatabaseError; + +// ACCOUNT HEADER +// ================================================================================================ + +#[derive(Debug, Clone, Queryable)] +struct AccountHeaderDataRaw { + code_commitment: Option>, + nonce: Option, + storage_header: Option>, +} + +/// Queries the account header for a specific account at a specific block number. +/// +/// This reconstructs the `AccountHeader` by reading from the `accounts` table: +/// - `account_id`, `nonce`, `code_commitment`, `storage_header`, `vault_root` +/// +/// Returns `None` if the account doesn't exist at that block. +/// +/// # Arguments +/// +/// * `conn` - Database connection +/// * `account_id` - The account ID to query +/// * `block_num` - The block number at which to query the account header +/// +/// # Returns +/// +/// * `Ok(Some(AccountHeader))` - The account header if found +/// * `Ok(None)` - If account doesn't exist at that block +/// * `Err(DatabaseError)` - If there's a database error +pub(crate) fn select_account_header_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { + use schema::accounts; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + + let account_data: Option<(AccountHeaderDataRaw, Option>)> = SelectDsl::select( + accounts::table + .filter(accounts::account_id.eq(&account_id_bytes)) + .filter(accounts::block_num.le(block_num_sql)) + .order(accounts::block_num.desc()) + .limit(1), + ( + (accounts::code_commitment, accounts::nonce, accounts::storage_header), + accounts::vault_root, + ), + ) + .first(conn) + .optional()?; + + let Some(( + AccountHeaderDataRaw { + code_commitment: code_commitment_bytes, + nonce: nonce_raw, + storage_header: storage_header_blob, + }, + vault_root_bytes, + )) = account_data + else { + return Ok(None); + }; + + let storage_commitment = match storage_header_blob { + Some(blob) => { + let storage = AccountStorage::read_from_bytes(&blob)?; + storage.to_commitment() + }, + None => Word::default(), + }; + + let code_commitment = code_commitment_bytes + .map(|bytes| Word::read_from_bytes(&bytes)) + .transpose()? + .unwrap_or(Word::default()); + + let nonce = nonce_raw.map_or(Felt::ZERO, raw_sql_to_nonce); + + let vault_root = vault_root_bytes + .map(|bytes| Word::read_from_bytes(&bytes)) + .transpose()? + .unwrap_or(Word::default()); + + Ok(Some(AccountHeader::new( + account_id, + nonce, + vault_root, + storage_commitment, + code_commitment, + ))) +} + +// ACCOUNT CODE +// ================================================================================================ + +/// Queries the account code for a specific account at a specific block number. +/// +/// Returns `None` if: +/// - The account doesn't exist at that block +/// - The account has no code (private account or account without code commitment) +/// +/// # Arguments +/// +/// * `conn` - Database connection +/// * `account_id` - The account ID to query +/// * `block_num` - The block number at which to query the account code +/// +/// # Returns +/// +/// * `Ok(Some(Vec))` - The account code bytes if found +/// * `Ok(None)` - If account doesn't exist or has no code +/// * `Err(DatabaseError)` - If there's a database error +pub(crate) fn select_account_code_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result>, DatabaseError> { + use schema::{account_codes, accounts}; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = i64::from(block_num.as_u32()); + // Query the accounts table to get the code_commitment at the specified block or earlier + // Then join with account_codes to get the actual code + let result: Option> = SelectDsl::select( + accounts::table + .inner_join(account_codes::table) + .filter(accounts::account_id.eq(&account_id_bytes)) + .filter(accounts::block_num.le(block_num_sql)) + .order(accounts::block_num.desc()) + .limit(1), + account_codes::code, + ) + .first(conn) + .optional()?; + + Ok(result) +} + +// ACCOUNT VAULT +// ================================================================================================ + +/// Query vault assets at a specific block by finding the most recent update for each `vault_key`. +pub(crate) fn select_account_vault_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { + use schema::account_vault_assets as t; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + + // Since Diesel doesn't support composite keys in subqueries easily, we use a two-step approach: + // Step 1: Get max block_num for each vault_key + let latest_blocks_per_vault_key = Vec::from_iter( + QueryDsl::select( + t::table + .filter(t::account_id.eq(&account_id_bytes)) + .filter(t::block_num.le(block_num_sql)) + .group_by(t::vault_key), + (t::vault_key, diesel::dsl::max(t::block_num)), + ) + .load::<(Vec, Option)>(conn)? + .into_iter() + .filter_map(|(key, maybe_block)| maybe_block.map(|block| (key, block))), + ); + + if latest_blocks_per_vault_key.is_empty() { + return Ok(Vec::new()); + } + + // Step 2: Fetch the full rows matching (vault_key, block_num) pairs + let mut assets = Vec::new(); + for (vault_key_bytes, max_block) in latest_blocks_per_vault_key { + let result: Option>> = QueryDsl::select( + t::table.filter( + t::account_id + .eq(&account_id_bytes) + .and(t::vault_key.eq(&vault_key_bytes)) + .and(t::block_num.eq(max_block)), + ), + t::asset, + ) + .first(conn) + .optional()?; + if let Some(Some(asset_bytes)) = result { + let asset = Asset::read_from_bytes(&asset_bytes)?; + assets.push(asset); + } + } + + // Sort by vault_key for consistent ordering + assets.sort_by_key(Asset::vault_key); + + Ok(assets) +} + +// ACCOUNT STORAGE +// ================================================================================================ + +/// Returns account storage header at a given block by reading from `accounts.storage_header` +/// and deserializing the storage header blob. +pub(crate) fn select_account_storage_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result { + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + + // Query storage blob for this account at this block + let storage_blob: Option> = + SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) + .filter(schema::accounts::account_id.eq(&account_id_bytes)) + .filter(schema::accounts::block_num.le(block_num_sql)) + .order(schema::accounts::block_num.desc()) + .limit(1) + .first(conn) + .optional()? + .flatten(); + + let Some(blob) = storage_blob else { + // No storage means empty storage + return Ok(AccountStorage::new(Vec::new())?); + }; + + // Deserialize the full AccountStorage from the blob + let storage = AccountStorage::read_from_bytes(&blob)?; + + Ok(storage) +} From 3009bf78478974283e083add225aa8a72c62d11f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 17:54:25 +0100 Subject: [PATCH 68/95] nope --- .../store/benches/account_tree_historical.rs | 2 +- crates/store/src/accounts/mod.rs | 5 +- crates/store/src/accounts/tests.rs | 2 +- crates/store/src/db/mod.rs | 9 + .../store/src/db/models/queries/accounts.rs | 102 +++- .../db/models/queries/accounts/at_block.rs | 81 ++- crates/store/src/db/tests.rs | 505 +++++++++++++++++- crates/store/src/errors.rs | 3 + crates/store/src/lib.rs | 4 +- crates/store/src/state.rs | 42 +- 10 files changed, 706 insertions(+), 49 deletions(-) diff --git a/crates/store/benches/account_tree_historical.rs b/crates/store/benches/account_tree_historical.rs index ba7a5c2cc3..8555a0ebff 100644 --- a/crates/store/benches/account_tree_historical.rs +++ b/crates/store/benches/account_tree_historical.rs @@ -1,7 +1,7 @@ use std::hint::black_box; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; -use miden_node_store::AccountTreeWithHistory; +use miden_node_store::accounts::AccountTreeWithHistory; use miden_protocol::Word; use miden_protocol::account::AccountId; use miden_protocol::block::BlockNumber; diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index c0a37be323..3ce4ff388d 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -8,7 +8,6 @@ use miden_protocol::block::account_tree::{AccountMutationSet, AccountTree, Accou use miden_protocol::crypto::merkle::smt::{ LargeSmt, LeafIndex, - MemoryStorage, NodeMutation, SMT_DEPTH, SmtLeaf, @@ -27,6 +26,7 @@ use miden_protocol::{AccountTreeError, EMPTY_WORD, Word}; mod tests; /// Convenience for an in-memory-only account tree. +#[cfg(test)] pub type InMemoryAccountTree = AccountTree>; // HISTORICAL ERROR TYPES @@ -62,7 +62,9 @@ enum HistoricalSelector { /// Captures reversion state for historical queries at a specific block. #[derive(Debug, Clone)] struct HistoricalOverlay { + #[allow(dead_code)] block_number: BlockNumber, + #[allow(dead_code)] root: Word, node_mutations: HashMap, account_updates: HashMap, (Word, Word)>, @@ -157,6 +159,7 @@ impl AccountTreeWithHistory { /// Returns the root hash at a specific historical block. /// /// Returns `None` if the block is in the future or too old (pruned). + #[cfg(test)] pub fn root_at(&self, block_number: BlockNumber) -> Option { match self.historical_selector(block_number) { HistoricalSelector::Latest => Some(self.latest.root()), diff --git a/crates/store/src/accounts/tests.rs b/crates/store/src/accounts/tests.rs index 5880d39825..f709289469 100644 --- a/crates/store/src/accounts/tests.rs +++ b/crates/store/src/accounts/tests.rs @@ -18,7 +18,7 @@ mod account_tree_with_history_tests { /// Helper function to create an `AccountTree` from entries using the new API fn create_account_tree( entries: impl IntoIterator, - ) -> AccountTree> { + ) -> InMemoryAccountTree { let smt_entries = entries .into_iter() .map(|(id, commitment)| (account_id_to_smt_key(id), commitment)); diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 01fda65c93..088847b9c5 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -400,6 +400,15 @@ impl Db { .await } + /// Returns all account IDs that have public state. + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] + pub async fn select_all_public_account_ids(&self) -> Result> { + self.transact("read all public account IDs", move |conn| { + queries::select_all_public_account_ids(conn) + }) + .await + } + /// Loads public account details from the DB. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_account(&self, id: AccountId) -> Result { diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 72b1bce745..52664d053e 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::ops::RangeInclusive; use diesel::prelude::{Queryable, QueryableByName}; @@ -30,9 +31,13 @@ use miden_protocol::account::{ AccountDelta, AccountId, AccountStorage, + AccountStorageHeader, NonFungibleDeltaAction, + StorageMap, + StorageSlot, StorageSlotContent, StorageSlotName, + StorageSlotType, }; use miden_protocol::asset::{Asset, AssetVault, AssetVaultKey, FungibleAsset}; use miden_protocol::block::{BlockAccountUpdate, BlockNumber}; @@ -243,6 +248,47 @@ pub(crate) fn select_all_account_commitments( )) } +/// Select all account IDs that have public state. +/// +/// This filters accounts in-memory after loading only the account IDs (not commitments), +/// which is more efficient than loading full commitments when only IDs are needed. +/// +/// # Raw SQL +/// +/// ```sql +/// SELECT +/// account_id +/// FROM +/// accounts +/// WHERE +/// is_latest = 1 +/// ORDER BY +/// block_num ASC +/// ``` +pub(crate) fn select_all_public_account_ids( + conn: &mut SqliteConnection, +) -> Result, DatabaseError> { + // We could technically use a `LIKE` constraint for both postgres and sqlite backends, + // but diesel doesn't expose that. + let raw: Vec> = + SelectDsl::select(schema::accounts::table, schema::accounts::account_id) + .filter(schema::accounts::is_latest.eq(true)) + .order_by(schema::accounts::block_num.asc()) + .load::>(conn)?; + + Result::from_iter( + raw.into_iter() + .map(|bytes| { + AccountId::read_from_bytes(&bytes).map_err(DatabaseError::DeserializationError) + }) + .filter_map(|result| match result { + Ok(id) if id.has_public_state() => Some(Ok(id)), + Ok(_) => None, + Err(e) => Some(Err(e)), + }), + ) +} + /// Select account vault assets within a block range (inclusive). /// /// # Parameters @@ -568,15 +614,18 @@ pub(crate) fn select_account_storage_map_values( Ok(StorageMapValuesPage { last_block_included, values }) } -/// Select latest account storage header by querying `accounts.storage_header` where -/// `is_latest=true`. +/// Select latest account storage by querying `accounts.storage_header` where `is_latest=true` +/// and reconstructing full storage from the header plus map values from +/// `account_storage_map_values`. pub(crate) fn select_latest_account_storage( conn: &mut SqliteConnection, account_id: AccountId, ) -> Result { + use schema::account_storage_map_values as t; + let account_id_bytes = account_id.to_bytes(); - // Query storage blob for this account where is_latest = true + // Query storage header blob for this account where is_latest = true let storage_blob: Option> = SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) .filter(schema::accounts::account_id.eq(&account_id_bytes)) @@ -590,10 +639,46 @@ pub(crate) fn select_latest_account_storage( return Ok(AccountStorage::new(Vec::new())?); }; - // Deserialize the full AccountStorage from the blob - let storage = AccountStorage::read_from_bytes(&blob)?; + // Deserialize the AccountStorageHeader from the blob + let header = AccountStorageHeader::read_from_bytes(&blob)?; + + // Query all latest map values for this account + let map_values: Vec<(String, Vec, Vec)> = + SelectDsl::select(t::table, (t::slot_name, t::key, t::value)) + .filter(t::account_id.eq(&account_id_bytes)) + .filter(t::is_latest.eq(true)) + .load(conn)?; - Ok(storage) + // Group map values by slot name + let mut map_entries_by_slot: BTreeMap> = BTreeMap::new(); + for (slot_name_str, key_bytes, value_bytes) in map_values { + let slot_name: StorageSlotName = slot_name_str.parse().map_err(|_| { + DatabaseError::DataCorrupted(format!("Invalid slot name: {slot_name_str}")) + })?; + let key = Word::read_from_bytes(&key_bytes)?; + let value = Word::read_from_bytes(&value_bytes)?; + map_entries_by_slot.entry(slot_name).or_default().push((key, value)); + } + + // Reconstruct StorageSlots from header slots + map entries + let mut slots = Vec::new(); + for slot_header in header.slots() { + let slot = match slot_header.slot_type() { + StorageSlotType::Value => { + // For value slots, the header value IS the slot value + StorageSlot::with_value(slot_header.name().clone(), slot_header.value()) + }, + StorageSlotType::Map => { + // For map slots, reconstruct from map entries + let entries = map_entries_by_slot.remove(slot_header.name()).unwrap_or_default(); + let storage_map = StorageMap::with_entries(entries)?; + StorageSlot::with_map(slot_header.name().clone(), storage_map) + }, + }; + slots.push(slot); + } + + Ok(AccountStorage::new(slots)?) } // ACCOUNT MUTATION @@ -870,7 +955,10 @@ pub(crate) fn upsert_accounts( code_commitment: full_account .as_ref() .map(|account| account.code().commitment().to_bytes()), - storage_header: full_account.as_ref().map(|account| account.storage().to_bytes()), + // Store only the header (slot metadata + map roots), not full storage with map contents + storage_header: full_account + .as_ref() + .map(|account| account.storage().to_header().to_bytes()), vault_root: full_account.as_ref().map(|account| account.vault().root().to_bytes()), is_latest: true, }; diff --git a/crates/store/src/db/models/queries/accounts/at_block.rs b/crates/store/src/db/models/queries/accounts/at_block.rs index 6ebad2531d..aaef34a159 100644 --- a/crates/store/src/db/models/queries/accounts/at_block.rs +++ b/crates/store/src/db/models/queries/accounts/at_block.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use diesel::prelude::Queryable; use diesel::query_dsl::methods::SelectDsl; use diesel::{ @@ -8,7 +10,16 @@ use diesel::{ RunQueryDsl, SqliteConnection, }; -use miden_protocol::account::{AccountHeader, AccountId, AccountStorage}; +use miden_protocol::account::{ + AccountHeader, + AccountId, + AccountStorage, + AccountStorageHeader, + StorageMap, + StorageSlot, + StorageSlotName, + StorageSlotType, +}; use miden_protocol::asset::Asset; use miden_protocol::block::BlockNumber; use miden_protocol::utils::{Deserializable, Serializable}; @@ -84,8 +95,8 @@ pub(crate) fn select_account_header_at_block( let storage_commitment = match storage_header_blob { Some(blob) => { - let storage = AccountStorage::read_from_bytes(&blob)?; - storage.to_commitment() + let header = AccountStorageHeader::read_from_bytes(&blob)?; + header.to_commitment() }, None => Word::default(), }; @@ -219,17 +230,20 @@ pub(crate) fn select_account_vault_at_block( // ACCOUNT STORAGE // ================================================================================================ -/// Returns account storage header at a given block by reading from `accounts.storage_header` -/// and deserializing the storage header blob. +/// Returns account storage at a given block by reading from `accounts.storage_header` +/// (which contains the `AccountStorageHeader`) and reconstructing full storage from +/// map values in `account_storage_map_values` table. pub(crate) fn select_account_storage_at_block( conn: &mut SqliteConnection, account_id: AccountId, block_num: BlockNumber, ) -> Result { + use schema::account_storage_map_values as t; + let account_id_bytes = account_id.to_bytes(); let block_num_sql = block_num.to_raw_sql(); - // Query storage blob for this account at this block + // Query storage header blob for this account at or before this block let storage_blob: Option> = SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) .filter(schema::accounts::account_id.eq(&account_id_bytes)) @@ -245,8 +259,57 @@ pub(crate) fn select_account_storage_at_block( return Ok(AccountStorage::new(Vec::new())?); }; - // Deserialize the full AccountStorage from the blob - let storage = AccountStorage::read_from_bytes(&blob)?; + // Deserialize the AccountStorageHeader from the blob + let header = AccountStorageHeader::read_from_bytes(&blob)?; + + // Query all map values for this account up to and including this block. + // For each (slot_name, key), we need the latest value at or before block_num. + // First, get all entries up to block_num + let map_values: Vec<(i64, String, Vec, Vec)> = + SelectDsl::select(t::table, (t::block_num, t::slot_name, t::key, t::value)) + .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.le(block_num_sql))) + .order((t::slot_name.asc(), t::key.asc(), t::block_num.desc())) + .load(conn)?; + + // For each (slot_name, key) pair, keep only the latest entry (highest block_num) + let mut latest_map_entries: BTreeMap<(StorageSlotName, Word), Word> = BTreeMap::new(); + + for (_, slot_name_str, key_bytes, value_bytes) in map_values { + let slot_name: StorageSlotName = slot_name_str.parse().map_err(|_| { + DatabaseError::DataCorrupted(format!("Invalid slot name: {slot_name_str}")) + })?; + let key = Word::read_from_bytes(&key_bytes)?; + + // Only insert if we haven't seen this (slot_name, key) yet + // (since results are ordered by block_num desc, first one is latest) + latest_map_entries + .entry((slot_name, key)) + .or_insert_with(|| Word::read_from_bytes(&value_bytes).unwrap_or_default()); + } + + // Group entries by slot name + let mut map_entries_by_slot: BTreeMap> = BTreeMap::new(); + for ((slot_name, key), value) in latest_map_entries { + map_entries_by_slot.entry(slot_name).or_default().push((key, value)); + } + + // Reconstruct StorageSlots from header slots + map entries + let mut slots = Vec::new(); + for slot_header in header.slots() { + let slot = match slot_header.slot_type() { + StorageSlotType::Value => { + // For value slots, the header value IS the slot value + StorageSlot::with_value(slot_header.name().clone(), slot_header.value()) + }, + StorageSlotType::Map => { + // For map slots, reconstruct from map entries + let entries = map_entries_by_slot.remove(slot_header.name()).unwrap_or_default(); + let storage_map = StorageMap::with_entries(entries)?; + StorageSlot::with_map(slot_header.name().clone(), storage_map) + }, + }; + slots.push(slot); + } - Ok(storage) + Ok(AccountStorage::new(slots)?) } diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 030f3a6a9c..488b9232de 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -12,6 +12,7 @@ use miden_protocol::account::delta::AccountUpdateDetails; use miden_protocol::account::{ Account, AccountBuilder, + AccountCode, AccountComponent, AccountDelta, AccountId, @@ -21,6 +22,7 @@ use miden_protocol::account::{ AccountType, AccountVaultDelta, StorageSlot, + StorageSlotContent, StorageSlotDelta, StorageSlotName, }; @@ -61,7 +63,7 @@ use miden_protocol::transaction::{ TransactionHeader, TransactionId, }; -use miden_protocol::utils::Serializable; +use miden_protocol::utils::{Deserializable, Serializable}; use miden_protocol::{EMPTY_WORD, Felt, FieldElement, Word, ZERO}; use miden_standards::account::auth::AuthRpoFalcon512; use miden_standards::code_builder::CodeBuilder; @@ -1864,3 +1866,504 @@ fn regression_1461_full_state_delta_inserts_vault_assets() { assert_eq!(vault_asset.asset, Some(expected_asset)); assert_eq!(vault_asset.vault_key, expected_asset.vault_key()); } + +// SERIALIZATION SYMMETRY TESTS +// ================================================================================================ +// +// These tests ensure that `to_bytes` and `from_bytes`/`read_from_bytes` are symmetric for all +// types used in database operations. This guarantees that data inserted into the database can +// always be correctly retrieved. + +#[test] +fn serialization_symmetry_core_types() { + // AccountId + let account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + let bytes = account_id.to_bytes(); + let restored = AccountId::read_from_bytes(&bytes).unwrap(); + assert_eq!(account_id, restored, "AccountId serialization must be symmetric"); + + // Word + let word = num_to_word(0x1234_5678_9ABC_DEF0); + let bytes = word.to_bytes(); + let restored = Word::read_from_bytes(&bytes).unwrap(); + assert_eq!(word, restored, "Word serialization must be symmetric"); + + // Nullifier + let nullifier = num_to_nullifier(0xDEAD_BEEF); + let bytes = nullifier.to_bytes(); + let restored = Nullifier::read_from_bytes(&bytes).unwrap(); + assert_eq!(nullifier, restored, "Nullifier serialization must be symmetric"); + + // TransactionId + let tx_id = TransactionId::new(num_to_word(1), num_to_word(2), num_to_word(3), num_to_word(4)); + let bytes = tx_id.to_bytes(); + let restored = TransactionId::read_from_bytes(&bytes).unwrap(); + assert_eq!(tx_id, restored, "TransactionId serialization must be symmetric"); + + // NoteId + let note_id = NoteId::new(num_to_word(1), num_to_word(2)); + let bytes = note_id.to_bytes(); + let restored = NoteId::read_from_bytes(&bytes).unwrap(); + assert_eq!(note_id, restored, "NoteId serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_block_header() { + let block_header = BlockHeader::new( + 1_u8.into(), + num_to_word(2), + 3.into(), + num_to_word(4), + num_to_word(5), + num_to_word(6), + num_to_word(7), + num_to_word(8), + num_to_word(9), + SecretKey::new().public_key(), + test_fee_params(), + 11_u8.into(), + ); + + let bytes = block_header.to_bytes(); + let restored = BlockHeader::read_from_bytes(&bytes).unwrap(); + assert_eq!(block_header, restored, "BlockHeader serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_assets() { + let faucet_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); + + // FungibleAsset + let fungible = FungibleAsset::new(faucet_id, 1000).unwrap(); + let asset: Asset = fungible.into(); + let bytes = asset.to_bytes(); + let restored = Asset::read_from_bytes(&bytes).unwrap(); + assert_eq!(asset, restored, "Asset (fungible) serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_account_code() { + let account = mock_account_code_and_storage( + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + [], + None, + ); + + let code = account.code(); + let bytes = code.to_bytes(); + let restored = AccountCode::read_from_bytes(&bytes).unwrap(); + assert_eq!(*code, restored, "AccountCode serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_sparse_merkle_path() { + let path = SparseMerklePath::default(); + let bytes = path.to_bytes(); + let restored = SparseMerklePath::read_from_bytes(&bytes).unwrap(); + assert_eq!(path, restored, "SparseMerklePath serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_note_metadata() { + let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + // Use a tag that roundtrips properly - NoteTag::LocalAny stores the full u32 including type + // bits + let tag = NoteTag::from_account_id(sender); + let metadata = NoteMetadata::new( + sender, + NoteType::Public, + tag, + NoteExecutionHint::always(), + Felt::new(42), + ) + .unwrap(); + + let bytes = metadata.to_bytes(); + let restored = NoteMetadata::read_from_bytes(&bytes).unwrap(); + assert_eq!(metadata, restored, "NoteMetadata serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_nullifier_vec() { + let nullifiers: Vec = (0..5).map(num_to_nullifier).collect(); + let bytes = nullifiers.to_bytes(); + let restored: Vec = Deserializable::read_from_bytes(&bytes).unwrap(); + assert_eq!(nullifiers, restored, "Vec serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_note_id_vec() { + let note_ids: Vec = + (0..5).map(|i| NoteId::new(num_to_word(i), num_to_word(i + 100))).collect(); + let bytes = note_ids.to_bytes(); + let restored: Vec = Deserializable::read_from_bytes(&bytes).unwrap(); + assert_eq!(note_ids, restored, "Vec serialization must be symmetric"); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_block_header() { + let mut conn = create_db(); + + let block_header = BlockHeader::new( + 1_u8.into(), + num_to_word(2), + BlockNumber::from(42), + num_to_word(4), + num_to_word(5), + num_to_word(6), + num_to_word(7), + num_to_word(8), + num_to_word(9), + SecretKey::new().public_key(), + test_fee_params(), + 11_u8.into(), + ); + + // Insert + queries::insert_block_header(&mut conn, &block_header).unwrap(); + + // Retrieve + let retrieved = + queries::select_block_header_by_block_num(&mut conn, Some(block_header.block_num())) + .unwrap() + .expect("Block header should exist"); + + assert_eq!(block_header, retrieved, "BlockHeader DB roundtrip must be symmetric"); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_nullifiers() { + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + let nullifiers: Vec = (0..5).map(|i| num_to_nullifier(i << 48)).collect(); + + // Insert + queries::insert_nullifiers_for_block(&mut conn, &nullifiers, block_num).unwrap(); + + // Retrieve + let retrieved = queries::select_all_nullifiers(&mut conn).unwrap(); + + assert_eq!(nullifiers.len(), retrieved.len(), "Should retrieve same number of nullifiers"); + for (orig, info) in nullifiers.iter().zip(retrieved.iter()) { + assert_eq!(*orig, info.nullifier, "Nullifier DB roundtrip must be symmetric"); + assert_eq!(block_num, info.block_num, "Block number must match"); + } +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_account() { + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + let account = mock_account_code_and_storage( + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + [], + Some([99u8; 32]), + ); + let account_id = account.id(); + let account_commitment = account.commitment(); + + // Insert with full delta (like genesis) + let account_delta = AccountDelta::try_from(account.clone()).unwrap(); + let block_update = BlockAccountUpdate::new( + account_id, + account_commitment, + AccountUpdateDetails::Delta(account_delta), + ); + queries::upsert_accounts(&mut conn, &[block_update], block_num).unwrap(); + + // Retrieve + let retrieved = queries::select_all_accounts(&mut conn).unwrap(); + assert_eq!(retrieved.len(), 1, "Should have one account"); + + let retrieved_info = &retrieved[0]; + assert_eq!( + retrieved_info.summary.account_id, account_id, + "AccountId DB roundtrip must be symmetric" + ); + assert_eq!( + retrieved_info.summary.account_commitment, account_commitment, + "Account commitment DB roundtrip must be symmetric" + ); + assert_eq!(retrieved_info.summary.block_num, block_num, "Block number must match"); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_notes() { + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + queries::upsert_accounts(&mut conn, &[mock_block_account_update(sender, 0)], block_num) + .unwrap(); + + let new_note = create_note(sender); + let note_index = BlockNoteIndex::new(0, 0).unwrap(); + + let note = NoteRecord { + block_num, + note_index, + note_id: new_note.id().as_word(), + note_commitment: new_note.commitment(), + metadata: *new_note.metadata(), + details: Some(NoteDetails::from(&new_note)), + inclusion_path: SparseMerklePath::default(), + }; + + // Insert + queries::insert_scripts(&mut conn, [¬e]).unwrap(); + queries::insert_notes(&mut conn, &[(note.clone(), None)]).unwrap(); + + // Retrieve + let note_ids = vec![NoteId::from_raw(note.note_id)]; + let retrieved = queries::select_notes_by_id(&mut conn, ¬e_ids).unwrap(); + + assert_eq!(retrieved.len(), 1, "Should have one note"); + let retrieved_note = &retrieved[0]; + + assert_eq!(note.note_id, retrieved_note.note_id, "NoteId DB roundtrip must be symmetric"); + assert_eq!( + note.note_commitment, retrieved_note.note_commitment, + "Note commitment DB roundtrip must be symmetric" + ); + assert_eq!( + note.metadata, retrieved_note.metadata, + "Metadata DB roundtrip must be symmetric" + ); + assert_eq!( + note.inclusion_path, retrieved_note.inclusion_path, + "Inclusion path DB roundtrip must be symmetric" + ); + assert_eq!( + note.details, retrieved_note.details, + "Note details DB roundtrip must be symmetric" + ); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_transactions() { + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + let account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + queries::upsert_accounts(&mut conn, &[mock_block_account_update(account_id, 1)], block_num) + .unwrap(); + + let tx = mock_block_transaction(account_id, 1); + let ordered_tx = OrderedTransactionHeaders::new_unchecked(vec![tx.clone()]); + + // Insert + queries::insert_transactions(&mut conn, block_num, &ordered_tx).unwrap(); + + // Retrieve + let retrieved = queries::select_transactions_by_accounts_and_block_range( + &mut conn, + &[account_id], + BlockNumber::from(0)..=BlockNumber::from(2), + ) + .unwrap(); + + assert_eq!(retrieved.len(), 1, "Should have one transaction"); + let retrieved_tx = &retrieved[0]; + + assert_eq!( + tx.account_id(), + retrieved_tx.account_id, + "AccountId DB roundtrip must be symmetric" + ); + assert_eq!( + tx.id(), + retrieved_tx.transaction_id, + "TransactionId DB roundtrip must be symmetric" + ); + assert_eq!(block_num, retrieved_tx.block_num, "Block number must match"); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_vault_assets() { + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + let faucet_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + + // Create account first + queries::upsert_accounts(&mut conn, &[mock_block_account_update(account_id, 0)], block_num) + .unwrap(); + + let fungible_asset = FungibleAsset::new(faucet_id, 5000).unwrap(); + let asset: Asset = fungible_asset.into(); + let vault_key = asset.vault_key(); + + // Insert vault asset + queries::insert_account_vault_asset(&mut conn, account_id, block_num, vault_key, Some(asset)) + .unwrap(); + + // Retrieve + let (_, vault_assets) = queries::select_account_vault_assets( + &mut conn, + account_id, + BlockNumber::GENESIS..=block_num, + ) + .unwrap(); + + assert_eq!(vault_assets.len(), 1, "Should have one vault asset"); + let retrieved = &vault_assets[0]; + + assert_eq!(retrieved.asset, Some(asset), "Asset DB roundtrip must be symmetric"); + assert_eq!(retrieved.vault_key, vault_key, "VaultKey DB roundtrip must be symmetric"); + assert_eq!(retrieved.block_num, block_num, "Block number must match"); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_storage_map_values() { + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + let slot_name = StorageSlotName::mock(5); + let key = num_to_word(12345); + let value = num_to_word(67890); + + // Insert + queries::insert_account_storage_map_value( + &mut conn, + account_id, + block_num, + slot_name.clone(), + key, + value, + ) + .unwrap(); + + // Retrieve + let page = queries::select_account_storage_map_values( + &mut conn, + account_id, + BlockNumber::GENESIS..=block_num, + ) + .unwrap(); + + assert_eq!(page.values.len(), 1, "Should have one storage map value"); + let retrieved = &page.values[0]; + + assert_eq!(retrieved.slot_name, slot_name, "StorageSlotName DB roundtrip must be symmetric"); + assert_eq!(retrieved.key, key, "Key (Word) DB roundtrip must be symmetric"); + assert_eq!(retrieved.value, value, "Value (Word) DB roundtrip must be symmetric"); + assert_eq!(retrieved.block_num, block_num, "Block number must match"); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_account_storage_with_maps() { + use miden_protocol::account::StorageMap; + + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + // Create storage with both value slots and map slots + let storage_map = StorageMap::with_entries(vec![ + ( + Word::from([Felt::new(1), Felt::ZERO, Felt::ZERO, Felt::ZERO]), + Word::from([Felt::new(10), Felt::new(20), Felt::new(30), Felt::new(40)]), + ), + ( + Word::from([Felt::new(2), Felt::ZERO, Felt::ZERO, Felt::ZERO]), + Word::from([Felt::new(50), Felt::new(60), Felt::new(70), Felt::new(80)]), + ), + ]) + .unwrap(); + + let component_storage = vec![ + StorageSlot::with_value(StorageSlotName::mock(0), num_to_word(42)), + StorageSlot::with_map(StorageSlotName::mock(1), storage_map), + StorageSlot::with_empty_value(StorageSlotName::mock(2)), + ]; + + let component_code = "pub proc foo push.1 end"; + let account_component_code = CodeBuilder::default() + .compile_component_code("test::interface", component_code) + .unwrap(); + let account_component = AccountComponent::new(account_component_code, component_storage) + .unwrap() + .with_supports_all_types(); + + let account = AccountBuilder::new([50u8; 32]) + .account_type(AccountType::RegularAccountUpdatableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(account_component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + let account_id = account.id(); + let original_storage = account.storage().clone(); + let original_commitment = original_storage.to_commitment(); + + // Insert the account (this should store header + map values separately) + let account_delta = AccountDelta::try_from(account.clone()).unwrap(); + let block_update = BlockAccountUpdate::new( + account_id, + account.commitment(), + AccountUpdateDetails::Delta(account_delta), + ); + queries::upsert_accounts(&mut conn, &[block_update], block_num).unwrap(); + + // Retrieve the storage using select_latest_account_storage (reconstructs from header + map + // values) + let retrieved_storage = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + let retrieved_commitment = retrieved_storage.to_commitment(); + + // Verify the commitment matches (this proves the reconstruction is correct) + assert_eq!( + original_commitment, retrieved_commitment, + "Storage commitment must match after DB roundtrip" + ); + + // Verify slot count matches + assert_eq!( + original_storage.slots().len(), + retrieved_storage.slots().len(), + "Number of slots must match" + ); + + // Verify each slot + for (original_slot, retrieved_slot) in + original_storage.slots().iter().zip(retrieved_storage.slots().iter()) + { + assert_eq!(original_slot.name(), retrieved_slot.name(), "Slot names must match"); + assert_eq!(original_slot.slot_type(), retrieved_slot.slot_type(), "Slot types must match"); + + match (original_slot.content(), retrieved_slot.content()) { + (StorageSlotContent::Value(orig), StorageSlotContent::Value(retr)) => { + assert_eq!(orig, retr, "Value slot contents must match"); + }, + (StorageSlotContent::Map(orig_map), StorageSlotContent::Map(retr_map)) => { + assert_eq!(orig_map.root(), retr_map.root(), "Map slot roots must match"); + for (key, value) in orig_map.entries() { + let retrieved_value = retr_map.get(key); + assert_eq!(*value, retrieved_value, "Map entry for key {:?} must match", key); + } + }, + // The slot_type assertion above guarantees matching variants, so this is unreachable + _ => unreachable!(), + } + } +} diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 7471c0b583..42a0fe32d9 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -22,6 +22,7 @@ use miden_protocol::{ FeeError, NoteError, NullifierTreeError, + StorageMapError, Word, }; use thiserror::Error; @@ -62,6 +63,8 @@ pub enum DatabaseError { NetworkAccountError(#[from] NetworkAccountError), #[error("note error")] NoteError(#[from] NoteError), + #[error("storage map error")] + StorageMapError(#[from] StorageMapError), #[error("setup deadpool connection pool failed")] Deadpool(#[from] deadpool::managed::PoolError), #[error("setup deadpool connection pool failed")] diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index ecfc66f148..633464e459 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -1,6 +1,6 @@ use std::time::Duration; -mod accounts; +pub mod accounts; mod blocks; mod db; mod errors; @@ -9,7 +9,7 @@ mod inner_forest; mod server; pub mod state; -pub use accounts::{AccountTreeWithHistory, HistoricalError, InMemoryAccountTree}; +pub(crate) use accounts::{AccountTreeWithHistory, HistoricalError}; pub use genesis::GenesisState; pub use server::{DataDirectory, Store}; diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index bfc526bc57..55534761b8 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -381,13 +381,18 @@ impl State { // Signals the write lock has been acquired, and the transaction can be committed let (inform_acquire_done, acquire_done) = oneshot::channel::<()>(); - // Extract account updates with deltas before block is moved into async task - // We'll use these deltas to update the SmtForest without DB roundtrips + // Extract public account updates with deltas before block is moved into async task. + // Private accounts are filtered out since they don't expose their state changes. let account_updates: Vec<_> = block .body() .updated_accounts() .iter() - .map(|update| (update.account_id(), update.details().clone())) + .filter_map(|update| match update.details() { + AccountUpdateDetails::Delta(delta) => { + Some((update.account_id(), AccountUpdateDetails::Delta(delta.clone()))) + }, + AccountUpdateDetails::Private => None, + }) .collect(); // The DB and in-memory state updates need to be synchronized and are partially @@ -456,20 +461,18 @@ impl State { Ok(()) } - /// Updates `SmtForest` with account deltas from a block - /// - /// This method updates the forest directly using the deltas extracted from the block. + /// Updates `SmtForest` with account deltas from a block. /// /// # Arguments /// - /// * `account_updates` - Vector of (`AccountId`, `AccountUpdateDetails`) tuples from the block + /// * `account_updates` - Vector of (`AccountId`, `AccountUpdateDetails`) tuples for public + /// accounts. Private accounts must be filtered out before calling this method. /// * `block_num` - Block number for which these updates apply /// /// # Note /// - /// - Private account updates are skipped as their state is not publicly visible. - /// - The number of changed accounts is implicitly bounded by the limited number of transactions - /// per block. + /// The number of changed accounts is implicitly bounded by the limited number of transactions + /// per block. #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = account_updates.len()))] async fn update_forest( &self, @@ -485,7 +488,6 @@ impl State { for (account_id, details) in account_updates { match details { AccountUpdateDetails::Delta(ref delta) => { - // Update the forest with the delta (handles both full-state and partial) forest_guard.update_account(block_num, delta); tracing::debug!( @@ -496,15 +498,7 @@ impl State { "Updated forest with account delta" ); }, - AccountUpdateDetails::Private => { - // Private accounts don't expose their state changes - tracing::trace!( - target: COMPONENT, - %account_id, - %block_num, - "Skipping private account update" - ); - }, + AccountUpdateDetails::Private => unreachable!("private accounts are filtered out"), } } @@ -1321,13 +1315,7 @@ async fn load_smt_forest( ) -> Result { use miden_protocol::account::delta::AccountDelta; - // Skip private accounts - they don't have public state to reconstruct - let public_account_ids: Vec = db - .select_all_account_commitments() - .await? - .iter() - .filter_map(|(id, _commitment)| if id.has_public_state() { Some(*id) } else { None }) - .collect(); + let public_account_ids = db.select_all_public_account_ids().await?; // Acquire write lock once for the entire initialization let mut forest = InnerForest::new(); From 3110962d9a217aa16ece865277031ae8ed20216d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 18:02:05 +0100 Subject: [PATCH 69/95] fix inconsistency --- crates/store/src/accounts/mod.rs | 2 +- crates/store/src/inner_forest/mod.rs | 23 +++++++++---- crates/store/src/inner_forest/tests.rs | 45 ++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index 3ce4ff388d..2e680fa947 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -8,6 +8,7 @@ use miden_protocol::block::account_tree::{AccountMutationSet, AccountTree, Accou use miden_protocol::crypto::merkle::smt::{ LargeSmt, LeafIndex, + MemoryStorage, NodeMutation, SMT_DEPTH, SmtLeaf, @@ -26,7 +27,6 @@ use miden_protocol::{AccountTreeError, EMPTY_WORD, Word}; mod tests; /// Convenience for an in-memory-only account tree. -#[cfg(test)] pub type InMemoryAccountTree = AccountTree>; // HISTORICAL ERROR TYPES diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 197d9e8727..0abe910042 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -46,16 +46,23 @@ impl InnerForest { *EmptySubtreeRoots::entry(SMT_DEPTH, 0) } - /// Retrieves the vault SMT root for an account at a given block, defaulting to empty. + /// Retrieves the vault SMT root for an account at or before the given block. + /// + /// Finds the most recent vault root entry for the account, since vault state persists + /// across blocks where no changes occur. fn get_vault_root(&self, account_id: AccountId, block_num: BlockNumber) -> Word { self.vault_roots - .get(&(account_id, block_num)) - .copied() + .range(..=(account_id, block_num)) + .rev() + .find(|((id, _), _)| *id == account_id) + .map(|(_, root)| *root) .unwrap_or_else(Self::empty_smt_root) } - /// Retrieves the storage map SMT root for an account slot at a given block, defaulting to - /// empty. + /// Retrieves the storage map SMT root for an account slot at or before the given block. + /// + /// Finds the most recent storage root entry for the slot, since storage state persists + /// across blocks where no changes occur. fn get_storage_root( &self, account_id: AccountId, @@ -63,8 +70,10 @@ impl InnerForest { block_num: BlockNumber, ) -> Word { self.storage_roots - .get(&(account_id, slot_name.clone(), block_num)) - .copied() + .range(..=(account_id, slot_name.clone(), block_num)) + .rev() + .find(|((id, name, _), _)| *id == account_id && name == slot_name) + .map(|(_, root)| *root) .unwrap_or_else(Self::empty_smt_root) } diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs index 046072a72e..7b27aefeac 100644 --- a/crates/store/src/inner_forest/tests.rs +++ b/crates/store/src/inner_forest/tests.rs @@ -202,3 +202,48 @@ fn test_full_state_delta_starts_from_empty_root() { // The full-state delta should produce the same root regardless of prior state assert_eq!(updated_root, fresh_root); } + +#[test] +fn test_vault_state_persists_across_blocks_without_changes() { + // Regression test for issue #7: vault state should persist across blocks + // where no changes occur, not reset to empty. + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); + + // Block 1: Add 100 tokens + let block_1 = BlockNumber::GENESIS.child(); + let mut vault_delta_1 = AccountVaultDelta::default(); + vault_delta_1.add_asset(dummy_fungible_asset(faucet_id, 100)).unwrap(); + let delta_1 = dummy_partial_delta(account_id, vault_delta_1, AccountStorageDelta::default()); + forest.update_account(block_1, &delta_1); + let root_after_block_1 = forest.vault_roots[&(account_id, block_1)]; + + // Blocks 2-5: No changes to this account (simulated by not calling update_account) + // This means no entries are added to vault_roots for these blocks. + + // Block 6: Add 50 more tokens + // The previous root lookup should find block_1's root, not return empty. + let block_6 = BlockNumber::from(6); + let mut vault_delta_6 = AccountVaultDelta::default(); + vault_delta_6.add_asset(dummy_fungible_asset(faucet_id, 150)).unwrap(); // 100 + 50 = 150 + let delta_6 = dummy_partial_delta(account_id, vault_delta_6, AccountStorageDelta::default()); + forest.update_account(block_6, &delta_6); + + // The root at block 6 should be different from block 1 (we added more tokens) + let root_after_block_6 = forest.vault_roots[&(account_id, block_6)]; + assert_ne!(root_after_block_1, root_after_block_6); + + // Verify get_vault_root finds the correct previous root for intermediate blocks + // Block 3 should return block 1's root (most recent before block 3) + let root_at_block_3 = forest.get_vault_root(account_id, BlockNumber::from(3)); + assert_eq!(root_at_block_3, root_after_block_1); + + // Block 5 should also return block 1's root + let root_at_block_5 = forest.get_vault_root(account_id, BlockNumber::from(5)); + assert_eq!(root_at_block_5, root_after_block_1); + + // Block 6 should return block 6's root + let root_at_block_6 = forest.get_vault_root(account_id, block_6); + assert_eq!(root_at_block_6, root_after_block_6); +} From 53cb5e810422367ed295a6f426851f69fa312552 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 18:19:34 +0100 Subject: [PATCH 70/95] faster --- crates/store/src/inner_forest/mod.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 0abe910042..99abe11c78 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -52,9 +52,8 @@ impl InnerForest { /// across blocks where no changes occur. fn get_vault_root(&self, account_id: AccountId, block_num: BlockNumber) -> Word { self.vault_roots - .range(..=(account_id, block_num)) - .rev() - .find(|((id, _), _)| *id == account_id) + .range((account_id, BlockNumber::GENESIS)..=(account_id, block_num)) + .next_back() .map(|(_, root)| *root) .unwrap_or_else(Self::empty_smt_root) } @@ -70,9 +69,11 @@ impl InnerForest { block_num: BlockNumber, ) -> Word { self.storage_roots - .range(..=(account_id, slot_name.clone(), block_num)) - .rev() - .find(|((id, name, _), _)| *id == account_id && name == slot_name) + .range( + (account_id, slot_name.clone(), BlockNumber::GENESIS) + ..=(account_id, slot_name.clone(), block_num), + ) + .next_back() .map(|(_, root)| *root) .unwrap_or_else(Self::empty_smt_root) } From 0cc0c61ca7aa9425263743bedc4aca8178cec9e0 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 18:50:54 +0100 Subject: [PATCH 71/95] undue changes --- crates/store/src/state.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 55534761b8..a458155eaf 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -30,7 +30,6 @@ use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_protocol::block::{ BlockHeader, BlockInputs, - BlockNoteTree, BlockNumber, Blockchain, ProvenBlock, @@ -332,14 +331,7 @@ impl State { }; // build note tree - let note_tree_entries = Vec::from_iter( - block - .body() - .output_notes() - .map(|(note_index, note)| (note_index, note.id(), *note.metadata())), - ); - let note_tree = BlockNoteTree::with_entries(note_tree_entries.iter().copied()) - .map_err(|e| InvalidBlockError::FailedToBuildNoteTree(e.to_string()))?; + let note_tree = block.body().compute_block_note_tree(); if note_tree.root() != header.note_root() { return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); } From ac7b8f9bc0e0037f40ef0794579056960d64b687 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 19:22:30 +0100 Subject: [PATCH 72/95] move fn to innerforest --- crates/store/src/inner_forest/mod.rs | 27 +++++++++++++++ crates/store/src/state.rs | 52 ++-------------------------- 2 files changed, 30 insertions(+), 49 deletions(-) diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 99abe11c78..c9c8276080 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -81,6 +81,33 @@ impl InnerForest { // PUBLIC INTERFACE // -------------------------------------------------------------------------------------------- + /// Applies account updates from a block to the forest. + /// + /// Iterates through account updates and applies each delta to the forest. + /// Private accounts should be filtered out before calling this method. + /// + /// # Arguments + /// + /// * `block_num` - Block number for which these updates apply + /// * `account_updates` - Iterator of (`AccountId`, `AccountDelta`) tuples for public accounts + pub(crate) fn apply_block_updates( + &mut self, + block_num: BlockNumber, + account_updates: impl IntoIterator, + ) { + for (account_id, delta) in account_updates { + self.update_account(block_num, &delta); + + tracing::debug!( + target: crate::COMPONENT, + %account_id, + %block_num, + is_full_state = delta.is_full_state(), + "Updated forest with account delta" + ); + } + } + /// Updates the forest with account vault and storage changes from a delta. /// /// Unified interface for updating all account state in the forest, handling both full-state diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index a458155eaf..982bbabe20 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -375,14 +375,12 @@ impl State { // Extract public account updates with deltas before block is moved into async task. // Private accounts are filtered out since they don't expose their state changes. - let account_updates: Vec<_> = block + let account_deltas: Vec<_> = block .body() .updated_accounts() .iter() .filter_map(|update| match update.details() { - AccountUpdateDetails::Delta(delta) => { - Some((update.account_id(), AccountUpdateDetails::Delta(delta.clone()))) - }, + AccountUpdateDetails::Delta(delta) => Some((update.account_id(), delta.clone())), AccountUpdateDetails::Private => None, }) .collect(); @@ -446,57 +444,13 @@ impl State { inner.blockchain.push(block_commitment); } - self.update_forest(account_updates, block_num).await?; + self.forest.write().await.apply_block_updates(block_num, account_deltas); info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); Ok(()) } - /// Updates `SmtForest` with account deltas from a block. - /// - /// # Arguments - /// - /// * `account_updates` - Vector of (`AccountId`, `AccountUpdateDetails`) tuples for public - /// accounts. Private accounts must be filtered out before calling this method. - /// * `block_num` - Block number for which these updates apply - /// - /// # Note - /// - /// The number of changed accounts is implicitly bounded by the limited number of transactions - /// per block. - #[instrument(target = COMPONENT, skip_all, fields(block_num = %block_num, num_accounts = account_updates.len()))] - async fn update_forest( - &self, - account_updates: Vec<(AccountId, AccountUpdateDetails)>, - block_num: BlockNumber, - ) -> Result<(), ApplyBlockError> { - if account_updates.is_empty() { - return Ok(()); - } - - let mut forest_guard = self.forest.write().await; - - for (account_id, details) in account_updates { - match details { - AccountUpdateDetails::Delta(ref delta) => { - forest_guard.update_account(block_num, delta); - - tracing::debug!( - target: COMPONENT, - %account_id, - %block_num, - is_full_state = delta.is_full_state(), - "Updated forest with account delta" - ); - }, - AccountUpdateDetails::Private => unreachable!("private accounts are filtered out"), - } - } - - Ok(()) - } - // STATE ACCESSORS // -------------------------------------------------------------------------------------------- From 369db2f15495a82d8147108b71f43ddfec1414d1 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 19:51:42 +0100 Subject: [PATCH 73/95] y --- crates/store/src/inner_forest/mod.rs | 40 ++++++++++++++++++--- crates/store/src/inner_forest/tests.rs | 50 ++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index c9c8276080..12b4fb5ea7 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -152,11 +152,41 @@ impl InnerForest { let mut entries = Vec::new(); // Process fungible assets - for (faucet_id, amount) in vault_delta.fungible().iter() { - let amount_u64: u64 = (*amount).try_into().expect("amount is non-negative"); - let asset: Asset = - FungibleAsset::new(*faucet_id, amount_u64).expect("valid fungible asset").into(); - entries.push((asset.vault_key().into(), Word::from(asset))); + for (faucet_id, amount_delta) in vault_delta.fungible().iter() { + let key: Word = FungibleAsset::new(*faucet_id, 0) + .expect("valid faucet id") + .vault_key() + .into(); + + let new_amount = if is_full_state { + // For full-state deltas, amount is the absolute value + (*amount_delta).try_into().expect("full-state amount should be non-negative") + } else { + // For partial deltas, amount is a change that must be applied to previous balance. + // + // TODO: SmtForest only exposes `fn open()` which computes a full Merkle + // proof. We only need the leaf, so a direct `fn get()` method would be faster. + let prev_amount = self + .forest + .open(prev_root, key) + .ok() + .and_then(|proof| proof.get(&key)) + .and_then(|word| FungibleAsset::try_from(word).ok()) + .map(|asset| asset.amount()) + .unwrap_or(0); + + let new_balance = (prev_amount as i128) + (*amount_delta as i128); + new_balance.max(0) as u64 + }; + + let value = if new_amount == 0 { + EMPTY_WORD + } else { + let asset: Asset = + FungibleAsset::new(*faucet_id, new_amount).expect("valid fungible asset").into(); + Word::from(asset) + }; + entries.push((key, value)); } // Process non-fungible assets diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs index 7b27aefeac..c8ce35d4d2 100644 --- a/crates/store/src/inner_forest/tests.rs +++ b/crates/store/src/inner_forest/tests.rs @@ -247,3 +247,53 @@ fn test_vault_state_persists_across_blocks_without_changes() { let root_at_block_6 = forest.get_vault_root(account_id, block_6); assert_eq!(root_at_block_6, root_after_block_6); } + +#[test] +fn test_partial_delta_applies_fungible_changes_correctly() { + // Regression test for issue #8: partial deltas should apply changes to previous balance, + // not treat amounts as absolute values. + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); + + // Block 1: Add 100 tokens (partial delta with +100) + let block_1 = BlockNumber::GENESIS.child(); + let mut vault_delta_1 = AccountVaultDelta::default(); + vault_delta_1.add_asset(dummy_fungible_asset(faucet_id, 100)).unwrap(); + let delta_1 = dummy_partial_delta(account_id, vault_delta_1, AccountStorageDelta::default()); + forest.update_account(block_1, &delta_1); + let root_after_100 = forest.vault_roots[&(account_id, block_1)]; + + // Block 2: Add 50 more tokens (partial delta with +50) + // Result should be 150 tokens, not 50 tokens + let block_2 = block_1.child(); + let mut vault_delta_2 = AccountVaultDelta::default(); + vault_delta_2.add_asset(dummy_fungible_asset(faucet_id, 50)).unwrap(); + let delta_2 = dummy_partial_delta(account_id, vault_delta_2, AccountStorageDelta::default()); + forest.update_account(block_2, &delta_2); + let root_after_150 = forest.vault_roots[&(account_id, block_2)]; + + // Roots should be different (100 tokens vs 150 tokens) + assert_ne!(root_after_100, root_after_150); + + // Block 3: Remove 30 tokens (partial delta with -30) + // Result should be 120 tokens + let block_3 = block_2.child(); + let mut vault_delta_3 = AccountVaultDelta::default(); + vault_delta_3.remove_asset(dummy_fungible_asset(faucet_id, 30)).unwrap(); + let delta_3 = dummy_partial_delta(account_id, vault_delta_3, AccountStorageDelta::default()); + forest.update_account(block_3, &delta_3); + let root_after_120 = forest.vault_roots[&(account_id, block_3)]; + + // Root should change again + assert_ne!(root_after_150, root_after_120); + + // Verify by creating a fresh forest with a full-state delta of 120 tokens + // The roots should match + let mut fresh_forest = InnerForest::new(); + let full_delta = dummy_full_state_delta(account_id, &[dummy_fungible_asset(faucet_id, 120)]); + fresh_forest.update_account(block_3, &full_delta); + let root_full_state_120 = fresh_forest.vault_roots[&(account_id, block_3)]; + + assert_eq!(root_after_120, root_full_state_120); +} From 6cd103323d7dd5df4d258b9bf8971cdf87f1cd21 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 20:08:06 +0100 Subject: [PATCH 74/95] another --- crates/store/src/inner_forest/mod.rs | 12 ++++--- crates/store/src/inner_forest/tests.rs | 43 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 12b4fb5ea7..253936855e 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -50,18 +50,23 @@ impl InnerForest { /// /// Finds the most recent vault root entry for the account, since vault state persists /// across blocks where no changes occur. + // + // TODO: a fallback to DB lookup is required once pruning lands. + // Currently returns empty root which would be incorrect fn get_vault_root(&self, account_id: AccountId, block_num: BlockNumber) -> Word { self.vault_roots .range((account_id, BlockNumber::GENESIS)..=(account_id, block_num)) .next_back() - .map(|(_, root)| *root) - .unwrap_or_else(Self::empty_smt_root) + .map_or_else(Self::empty_smt_root, |(_, root)| *root) } /// Retrieves the storage map SMT root for an account slot at or before the given block. /// /// Finds the most recent storage root entry for the slot, since storage state persists /// across blocks where no changes occur. + // + // TODO: a fallback to DB lookup is required once pruning lands. + // Currently returns empty root which would be incorrect fn get_storage_root( &self, account_id: AccountId, @@ -74,8 +79,7 @@ impl InnerForest { ..=(account_id, slot_name.clone(), block_num), ) .next_back() - .map(|(_, root)| *root) - .unwrap_or_else(Self::empty_smt_root) + .map_or_else(Self::empty_smt_root, |(_, root)| *root) } // PUBLIC INTERFACE diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs index c8ce35d4d2..a24b92fd88 100644 --- a/crates/store/src/inner_forest/tests.rs +++ b/crates/store/src/inner_forest/tests.rs @@ -297,3 +297,46 @@ fn test_partial_delta_applies_fungible_changes_correctly() { assert_eq!(root_after_120, root_full_state_120); } + +#[test] +fn test_partial_delta_across_long_block_range() { + // Validation test: partial deltas should work across 101+ blocks. + // + // This test passes now because InnerForest keeps all history. Once pruning is implemented + // (estimated ~50 blocks), this test will fail unless DB fallback is also implemented. + // When that happens, the test should be updated to use DB fallback or converted to an + // integration test that has DB access. + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let faucet_id = dummy_faucet(); + + // Block 1: Add 1000 tokens + let block_1 = BlockNumber::GENESIS.child(); + let mut vault_delta_1 = AccountVaultDelta::default(); + vault_delta_1.add_asset(dummy_fungible_asset(faucet_id, 1000)).unwrap(); + let delta_1 = dummy_partial_delta(account_id, vault_delta_1, AccountStorageDelta::default()); + forest.update_account(block_1, &delta_1); + let root_after_1000 = forest.vault_roots[&(account_id, block_1)]; + + // Blocks 2-100: No changes to this account (simulating long gap) + + // Block 101: Add 500 more tokens (partial delta with +500) + // This requires looking up block 1's state across a 100-block gap. + let block_101 = BlockNumber::from(101); + let mut vault_delta_101 = AccountVaultDelta::default(); + vault_delta_101.add_asset(dummy_fungible_asset(faucet_id, 500)).unwrap(); + let delta_101 = dummy_partial_delta(account_id, vault_delta_101, AccountStorageDelta::default()); + forest.update_account(block_101, &delta_101); + let root_after_1500 = forest.vault_roots[&(account_id, block_101)]; + + // Roots should be different (1000 tokens vs 1500 tokens) + assert_ne!(root_after_1000, root_after_1500); + + // Verify the final state matches a fresh forest with 1500 tokens + let mut fresh_forest = InnerForest::new(); + let full_delta = dummy_full_state_delta(account_id, &[dummy_fungible_asset(faucet_id, 1500)]); + fresh_forest.update_account(block_101, &full_delta); + let root_full_state_1500 = fresh_forest.vault_roots[&(account_id, block_101)]; + + assert_eq!(root_after_1500, root_full_state_1500); +} From 7613624d50ff7e9ffa1c3e9225721b59ff1a198b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 20:28:56 +0100 Subject: [PATCH 75/95] test re-review --- crates/store/src/inner_forest/mod.rs | 18 +++--- crates/store/src/inner_forest/tests.rs | 89 ++++++++++++++++++++++++-- crates/store/src/state.rs | 8 +-- 3 files changed, 91 insertions(+), 24 deletions(-) diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 253936855e..f7b161da77 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -157,10 +157,8 @@ impl InnerForest { // Process fungible assets for (faucet_id, amount_delta) in vault_delta.fungible().iter() { - let key: Word = FungibleAsset::new(*faucet_id, 0) - .expect("valid faucet id") - .vault_key() - .into(); + let key: Word = + FungibleAsset::new(*faucet_id, 0).expect("valid faucet id").vault_key().into(); let new_amount = if is_full_state { // For full-state deltas, amount is the absolute value @@ -176,18 +174,18 @@ impl InnerForest { .ok() .and_then(|proof| proof.get(&key)) .and_then(|word| FungibleAsset::try_from(word).ok()) - .map(|asset| asset.amount()) - .unwrap_or(0); + .map_or(0, |asset| asset.amount()); - let new_balance = (prev_amount as i128) + (*amount_delta as i128); - new_balance.max(0) as u64 + let new_balance = i128::from(prev_amount) + i128::from(*amount_delta); + u64::try_from(new_balance.max(0)).expect("balance fits in u64") }; let value = if new_amount == 0 { EMPTY_WORD } else { - let asset: Asset = - FungibleAsset::new(*faucet_id, new_amount).expect("valid fungible asset").into(); + let asset: Asset = FungibleAsset::new(*faucet_id, new_amount) + .expect("valid fungible asset") + .into(); Word::from(asset) }; entries.push((key, value)); diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs index a24b92fd88..d7351b892a 100644 --- a/crates/store/src/inner_forest/tests.rs +++ b/crates/store/src/inner_forest/tests.rs @@ -139,12 +139,6 @@ fn test_compare_partial_vs_full_state_delta_vault() { assert_ne!(*root_partial, EMPTY_WORD); } -#[test] -fn test_slot_names_are_tracked() { - let forest = InnerForest::new(); - let _: &BTreeMap<(AccountId, StorageSlotName, BlockNumber), Word> = &forest.storage_roots; -} - #[test] fn test_incremental_vault_updates() { let mut forest = InnerForest::new(); @@ -325,7 +319,8 @@ fn test_partial_delta_across_long_block_range() { let block_101 = BlockNumber::from(101); let mut vault_delta_101 = AccountVaultDelta::default(); vault_delta_101.add_asset(dummy_fungible_asset(faucet_id, 500)).unwrap(); - let delta_101 = dummy_partial_delta(account_id, vault_delta_101, AccountStorageDelta::default()); + let delta_101 = + dummy_partial_delta(account_id, vault_delta_101, AccountStorageDelta::default()); forest.update_account(block_101, &delta_101); let root_after_1500 = forest.vault_roots[&(account_id, block_101)]; @@ -340,3 +335,83 @@ fn test_partial_delta_across_long_block_range() { assert_eq!(root_after_1500, root_full_state_1500); } + +#[test] +fn test_update_storage_map() { + use std::collections::BTreeMap; + + use miden_protocol::account::delta::{StorageMapDelta, StorageSlotDelta}; + + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + let block_num = BlockNumber::GENESIS.child(); + + let slot_name = StorageSlotName::mock(3); + let key = Word::from([1u32, 2, 3, 4]); + let value = Word::from([5u32, 6, 7, 8]); + + let mut map_delta = StorageMapDelta::default(); + map_delta.insert(key, value); + let raw = BTreeMap::from_iter([(slot_name.clone(), StorageSlotDelta::Map(map_delta))]); + let storage_delta = AccountStorageDelta::from_raw(raw); + + let delta = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta); + forest.update_account(block_num, &delta); + + // Verify storage root was created + assert!(forest.storage_roots.contains_key(&(account_id, slot_name.clone(), block_num))); + let storage_root = forest.storage_roots[&(account_id, slot_name, block_num)]; + assert_ne!(storage_root, InnerForest::empty_smt_root()); +} + +#[test] +fn test_storage_map_incremental_updates() { + use std::collections::BTreeMap; + + use miden_protocol::account::delta::{StorageMapDelta, StorageSlotDelta}; + + let mut forest = InnerForest::new(); + let account_id = dummy_account(); + + let slot_name = StorageSlotName::mock(3); + let key1 = Word::from([1u32, 0, 0, 0]); + let key2 = Word::from([2u32, 0, 0, 0]); + let value1 = Word::from([10u32, 0, 0, 0]); + let value2 = Word::from([20u32, 0, 0, 0]); + let value3 = Word::from([30u32, 0, 0, 0]); + + // Block 1: Insert key1 -> value1 + let block_1 = BlockNumber::GENESIS.child(); + let mut map_delta_1 = StorageMapDelta::default(); + map_delta_1.insert(key1, value1); + let raw_1 = BTreeMap::from_iter([(slot_name.clone(), StorageSlotDelta::Map(map_delta_1))]); + let storage_delta_1 = AccountStorageDelta::from_raw(raw_1); + let delta_1 = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta_1); + forest.update_account(block_1, &delta_1); + let root_1 = forest.storage_roots[&(account_id, slot_name.clone(), block_1)]; + + // Block 2: Insert key2 -> value2 (key1 should persist) + let block_2 = block_1.child(); + let mut map_delta_2 = StorageMapDelta::default(); + map_delta_2.insert(key2, value2); + let raw_2 = BTreeMap::from_iter([(slot_name.clone(), StorageSlotDelta::Map(map_delta_2))]); + let storage_delta_2 = AccountStorageDelta::from_raw(raw_2); + let delta_2 = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta_2); + forest.update_account(block_2, &delta_2); + let root_2 = forest.storage_roots[&(account_id, slot_name.clone(), block_2)]; + + // Block 3: Update key1 -> value3 + let block_3 = block_2.child(); + let mut map_delta_3 = StorageMapDelta::default(); + map_delta_3.insert(key1, value3); + let raw_3 = BTreeMap::from_iter([(slot_name.clone(), StorageSlotDelta::Map(map_delta_3))]); + let storage_delta_3 = AccountStorageDelta::from_raw(raw_3); + let delta_3 = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta_3); + forest.update_account(block_3, &delta_3); + let root_3 = forest.storage_roots[&(account_id, slot_name, block_3)]; + + // All roots should be different + assert_ne!(root_1, root_2); + assert_ne!(root_2, root_3); + assert_ne!(root_1, root_3); +} diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 982bbabe20..47ac190f9e 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -27,13 +27,7 @@ use miden_protocol::account::delta::AccountUpdateDetails; use miden_protocol::account::{AccountId, StorageSlotContent}; use miden_protocol::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; -use miden_protocol::block::{ - BlockHeader, - BlockInputs, - BlockNumber, - Blockchain, - ProvenBlock, -}; +use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, ProvenBlock}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; use miden_protocol::crypto::merkle::smt::{ LargeSmt, From e04ff10e951d329f19cdc2809240f487195f32fb Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 20:43:38 +0100 Subject: [PATCH 76/95] fuckup --- crates/store/src/db/models/queries/accounts.rs | 2 +- crates/store/src/state.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 52664d053e..bec7664dea 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -95,7 +95,7 @@ pub(crate) fn select_account( // Backfill account details from database // For private accounts, we don't store full details in the database - let details = if account_id.is_public() { + let details = if account_id.has_public_state() { Some(select_full_account(conn, account_id)?) } else { None diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 47ac190f9e..affd197910 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -952,14 +952,14 @@ impl State { ) -> Result { let AccountProofRequest { block_num, account_id, details } = account_request; - if details.is_some() && !account_id.is_public() { + if details.is_some() && !account_id.has_public_state() { return Err(DatabaseError::AccountNotPublic(account_id)); } let (block_num, witness) = self.get_account_witness(block_num, account_id).await?; let details = if let Some(request) = details { - Some(self.fetch_requested_account_details(account_id, block_num, request).await?) + Some(self.fetch_public_account_details(account_id, block_num, request).await?) } else { None }; @@ -1005,7 +1005,7 @@ impl State { /// /// This method queries the database to fetch the account state and processes the detail /// request to return only the requested information. - async fn fetch_requested_account_details( + async fn fetch_public_account_details( &self, account_id: AccountId, block_num: BlockNumber, @@ -1017,7 +1017,7 @@ impl State { storage_requests, } = detail_request; - if !account_id.is_public() { + if !account_id.has_public_state() { return Err(DatabaseError::AccountNotPublic(account_id)); } From 9d8c2203635e16dcfd414b0ddc6b1cd2c976e48b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 20:57:07 +0100 Subject: [PATCH 77/95] update --- crates/store/src/inner_forest/mod.rs | 6 +++--- crates/store/src/state.rs | 16 +++++++--------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index f7b161da77..2ffae64310 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -97,14 +97,14 @@ impl InnerForest { pub(crate) fn apply_block_updates( &mut self, block_num: BlockNumber, - account_updates: impl IntoIterator, + account_updates: impl IntoIterator, ) { - for (account_id, delta) in account_updates { + for delta in account_updates { self.update_account(block_num, &delta); tracing::debug!( target: crate::COMPONENT, - %account_id, + account_id = %delta.id(), %block_num, is_full_state = delta.is_full_state(), "Updated forest with account delta" diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index affd197910..242c2401d5 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -369,15 +369,13 @@ impl State { // Extract public account updates with deltas before block is moved into async task. // Private accounts are filtered out since they don't expose their state changes. - let account_deltas: Vec<_> = block - .body() - .updated_accounts() - .iter() - .filter_map(|update| match update.details() { - AccountUpdateDetails::Delta(delta) => Some((update.account_id(), delta.clone())), - AccountUpdateDetails::Private => None, - }) - .collect(); + let account_deltas = + Vec::from_iter(block.body().updated_accounts().iter().filter_map(|update| { + match update.details() { + AccountUpdateDetails::Delta(delta) => Some(delta.clone()), + AccountUpdateDetails::Private => None, + } + })); // The DB and in-memory state updates need to be synchronized and are partially // overlapping. Namely, the DB transaction only proceeds after this task acquires the From 5ed1a4f16e3bb219a3ef377de665cb2d31a4e359 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 21:37:37 +0100 Subject: [PATCH 78/95] sync docs --- crates/store/src/inner_forest/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 2ffae64310..5778091af0 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -93,7 +93,7 @@ impl InnerForest { /// # Arguments /// /// * `block_num` - Block number for which these updates apply - /// * `account_updates` - Iterator of (`AccountId`, `AccountDelta`) tuples for public accounts + /// * `account_updates` - Iterator of `AccountDelta` for public accounts pub(crate) fn apply_block_updates( &mut self, block_num: BlockNumber, From c43a5a24bd2a435bc05b5cfba6218c7bcb7207e0 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 29 Dec 2025 22:30:03 +0100 Subject: [PATCH 79/95] splits --- crates/proto/src/domain/account.rs | 239 +++--- .../db/migrations/2025062000000_setup/up.sql | 8 +- crates/store/src/db/mod.rs | 92 ++- crates/store/src/db/models/conv.rs | 33 +- .../store/src/db/models/queries/accounts.rs | 486 ++++++------ .../db/models/queries/accounts/at_block.rs | 315 ++++++++ .../src/db/models/queries/accounts/tests.rs | 478 ++++++++++++ crates/store/src/db/schema.rs | 4 +- crates/store/src/db/tests.rs | 731 +++++++++++++++++- crates/store/src/errors.rs | 22 +- crates/store/src/state.rs | 202 +++-- 11 files changed, 2156 insertions(+), 454 deletions(-) create mode 100644 crates/store/src/db/models/queries/accounts/at_block.rs create mode 100644 crates/store/src/db/models/queries/accounts/tests.rs diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 2ef2be02c6..4c11a44789 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -1,7 +1,6 @@ use std::fmt::{Debug, Display, Formatter}; use miden_node_utils::formatting::format_opt; -use miden_protocol::Word; use miden_protocol::account::{ Account, AccountHeader, @@ -18,6 +17,7 @@ use miden_protocol::block::account_tree::AccountWitness; use miden_protocol::crypto::merkle::SparseMerklePath; use miden_protocol::note::{NoteExecutionMode, NoteTag}; use miden_protocol::utils::{Deserializable, DeserializationError, Serializable}; +use miden_protocol::{AssetError, Word}; use thiserror::Error; use super::try_convert; @@ -72,6 +72,7 @@ impl From for proto::account::AccountId { // ACCOUNT UPDATE // ================================================================================================ +// TODO should be called `AccountStateRef` or so #[derive(Debug, PartialEq)] pub struct AccountSummary { pub account_id: AccountId, @@ -99,7 +100,7 @@ impl From<&AccountInfo> for proto::account::AccountDetails { fn from(AccountInfo { summary, details }: &AccountInfo) -> Self { Self { summary: Some(summary.into()), - details: details.as_ref().map(miden_protocol::utils::Serializable::to_bytes), + details: details.as_ref().map(Serializable::to_bytes), } } } @@ -192,6 +193,7 @@ impl TryFrom fn try_from( value: proto::rpc::account_storage_details::AccountStorageMapDetails, ) -> Result { + use proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry; let proto::rpc::account_storage_details::AccountStorageMapDetails { slot_name, too_many_entries, @@ -200,32 +202,32 @@ impl TryFrom let slot_name = StorageSlotName::new(slot_name)?; - // Extract map_entries from the MapEntries message - let map_entries = if let Some(entries) = entries { - entries - .entries - .into_iter() - .map(|entry| { - let key = entry - .key - .ok_or(proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry::missing_field( - stringify!(key), - ))? - .try_into()?; - let value = entry - .value - .ok_or(proto::rpc::account_storage_details::account_storage_map_details::map_entries::StorageMapEntry::missing_field( - stringify!(value), - ))? - .try_into()?; - Ok((key, value)) - }) - .collect::, ConversionError>>()? + let entries = if too_many_entries { + StorageMapEntries::LimitExceeded } else { - Vec::new() + let map_entries = if let Some(entries) = entries { + entries + .entries + .into_iter() + .map(|entry| { + let key = entry + .key + .ok_or(StorageMapEntry::missing_field(stringify!(key)))? + .try_into()?; + let value = entry + .value + .ok_or(StorageMapEntry::missing_field(stringify!(value)))? + .try_into()?; + Ok((key, value)) + }) + .collect::, ConversionError>>()? + } else { + Vec::new() + }; + StorageMapEntries::Entries(map_entries) }; - Ok(Self { slot_name, too_many_entries, map_entries }) + Ok(Self { slot_name, entries }) } } @@ -346,37 +348,64 @@ impl From for proto::account::AccountStorageHeader { } } +/// Account vault details +/// +/// When an account contains a large number of assets (> +/// [`AccountVaultDetails::MAX_RETURN_ENTRIES`]), including all assets in a single RPC response +/// creates performance issues. In such cases, the `LimitExceeded` variant indicates to the client +/// to use the `SyncAccountVault` endpoint instead. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct AccountVaultDetails { - pub too_many_assets: bool, - pub assets: Vec, +pub enum AccountVaultDetails { + /// The vault has too many assets to return inline. + /// Clients must use `SyncAccountVault` endpoint instead. + LimitExceeded, + + /// The assets in the vault (up to `MAX_RETURN_ENTRIES`). + Assets(Vec), } + impl AccountVaultDetails { - const MAX_RETURN_ENTRIES: usize = 1000; + /// Maximum number of vault entries that can be returned in a single response. + /// Accounts with more assets will have `LimitExceeded` variant. + pub const MAX_RETURN_ENTRIES: usize = 1000; pub fn new(vault: &AssetVault) -> Self { if vault.assets().nth(Self::MAX_RETURN_ENTRIES).is_some() { - Self::too_many() + Self::LimitExceeded } else { - Self { - too_many_assets: false, - assets: Vec::from_iter(vault.assets()), - } + Self::Assets(Vec::from_iter(vault.assets())) } } pub fn empty() -> Self { - Self { - too_many_assets: false, - assets: Vec::new(), + Self::Assets(Vec::new()) + } + + /// Creates `AccountVaultDetails` from a list of assets. + pub fn from_assets(assets: Vec) -> Self { + if assets.len() > Self::MAX_RETURN_ENTRIES { + Self::LimitExceeded + } else { + Self::Assets(assets) } } - fn too_many() -> Self { - Self { - too_many_assets: true, - assets: Vec::new(), + /// Creates `AccountVaultDetails` from vault entries (key-value pairs). + /// + /// This is useful when entries have been fetched directly from the database + /// rather than extracted from an `AssetVault`. + /// + /// The entries are `(vault_key, asset)` pairs where `asset` is a Word representation. + pub fn from_entries(entries: Vec<(Word, Word)>) -> Result { + if entries.len() > Self::MAX_RETURN_ENTRIES { + return Ok(Self::LimitExceeded); } + + let assets = Result::, _>::from_iter( + entries.into_iter().map(|(_key, asset_word)| Asset::try_from(asset_word)), + )?; + + Ok(Self::Assets(assets)) } } @@ -386,40 +415,66 @@ impl TryFrom for AccountVaultDetails { fn try_from(value: proto::rpc::AccountVaultDetails) -> Result { let proto::rpc::AccountVaultDetails { too_many_assets, assets } = value; - let assets = - Result::, ConversionError>::from_iter(assets.into_iter().map(|asset| { - let asset = asset - .asset - .ok_or(proto::primitives::Asset::missing_field(stringify!(asset)))?; - let asset = Word::try_from(asset)?; - Asset::try_from(asset).map_err(ConversionError::AssetError) - }))?; - Ok(Self { too_many_assets, assets }) + if too_many_assets { + Ok(Self::LimitExceeded) + } else { + let parsed_assets = + Result::, ConversionError>::from_iter(assets.into_iter().map(|asset| { + let asset = asset + .asset + .ok_or(proto::primitives::Asset::missing_field(stringify!(asset)))?; + let asset = Word::try_from(asset)?; + Asset::try_from(asset).map_err(ConversionError::AssetError) + }))?; + Ok(Self::Assets(parsed_assets)) + } } } impl From for proto::rpc::AccountVaultDetails { fn from(value: AccountVaultDetails) -> Self { - let AccountVaultDetails { too_many_assets, assets } = value; - - Self { - too_many_assets, - assets: Vec::from_iter(assets.into_iter().map(|asset| proto::primitives::Asset { - asset: Some(proto::primitives::Digest::from(Word::from(asset))), - })), + match value { + AccountVaultDetails::LimitExceeded => Self { + too_many_assets: true, + assets: Vec::new(), + }, + AccountVaultDetails::Assets(assets) => Self { + too_many_assets: false, + assets: Vec::from_iter(assets.into_iter().map(|asset| proto::primitives::Asset { + asset: Some(proto::primitives::Digest::from(Word::from(asset))), + })), + }, } } } +/// Storage map entries for an account storage slot. +/// +/// When a storage map contains many entries (> [`AccountStorageMapDetails::MAX_RETURN_ENTRIES`]), +/// returning all entries in a single RPC response creates performance issues. In such cases, +/// the `LimitExceeded` variant indicates to the client to use the `SyncStorageMaps` endpoint +/// instead. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StorageMapEntries { + /// The map has too many entries to return inline. + /// Clients must use `SyncStorageMaps` endpoint instead. + LimitExceeded, + + /// The storage map entries (key-value pairs), up to `MAX_RETURN_ENTRIES`. + /// TODO: For partial responses, also include Merkle proofs and inner SMT nodes. + Entries(Vec<(Word, Word)>), +} + +/// Details about an account storage map slot. #[derive(Debug, Clone, PartialEq, Eq)] pub struct AccountStorageMapDetails { pub slot_name: StorageSlotName, - pub too_many_entries: bool, - pub map_entries: Vec<(Word, Word)>, + pub entries: StorageMapEntries, } impl AccountStorageMapDetails { - const MAX_RETURN_ENTRIES: usize = 1000; + /// Maximum number of storage map entries that can be returned in a single response. + pub const MAX_RETURN_ENTRIES: usize = 1000; pub fn new(slot_name: StorageSlotName, slot_data: SlotData, storage_map: &StorageMap) -> Self { match slot_data { @@ -430,13 +485,15 @@ impl AccountStorageMapDetails { fn from_all_entries(slot_name: StorageSlotName, storage_map: &StorageMap) -> Self { if storage_map.num_entries() > Self::MAX_RETURN_ENTRIES { - Self::too_many_entries(slot_name) + Self { + slot_name, + entries: StorageMapEntries::LimitExceeded, + } } else { let map_entries = Vec::from_iter(storage_map.entries().map(|(k, v)| (*k, *v))); Self { slot_name, - too_many_entries: false, - map_entries, + entries: StorageMapEntries::Entries(map_entries), } } } @@ -447,20 +504,15 @@ impl AccountStorageMapDetails { storage_map: &StorageMap, ) -> Self { if keys.len() > Self::MAX_RETURN_ENTRIES { - Self::too_many_entries(slot_name) + Self { + slot_name, + entries: StorageMapEntries::LimitExceeded, + } } else { // TODO For now, we return all entries instead of specific keys with proofs Self::from_all_entries(slot_name, storage_map) } } - - pub fn too_many_entries(slot_name: StorageSlotName) -> Self { - Self { - slot_name, - too_many_entries: true, - map_entries: Vec::new(), - } - } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -498,16 +550,16 @@ impl From for proto::rpc::AccountStorageDetails { const fn storage_slot_type_from_raw(slot_type: u32) -> Result { Ok(match slot_type { - 0 => StorageSlotType::Map, - 1 => StorageSlotType::Value, + 0 => StorageSlotType::Value, + 1 => StorageSlotType::Map, _ => return Err(ConversionError::EnumDiscriminantOutOfRange), }) } const fn storage_slot_type_to_raw(slot_type: StorageSlotType) -> u32 { match slot_type { - StorageSlotType::Map => 0, - StorageSlotType::Value => 1, + StorageSlotType::Value => 0, + StorageSlotType::Map => 1, } } @@ -628,21 +680,30 @@ impl From fn from(value: AccountStorageMapDetails) -> Self { use proto::rpc::account_storage_details::account_storage_map_details; - let AccountStorageMapDetails { slot_name, too_many_entries, map_entries } = value; + let AccountStorageMapDetails { slot_name, entries } = value; - let entries = Some(account_storage_map_details::MapEntries { - entries: Vec::from_iter(map_entries.into_iter().map(|(key, value)| { - account_storage_map_details::map_entries::StorageMapEntry { - key: Some(key.into()), - value: Some(value.into()), + match entries { + StorageMapEntries::LimitExceeded => Self { + slot_name: slot_name.to_string(), + too_many_entries: true, + entries: Some(account_storage_map_details::MapEntries { entries: Vec::new() }), + }, + StorageMapEntries::Entries(map_entries) => { + let entries = Some(account_storage_map_details::MapEntries { + entries: Vec::from_iter(map_entries.into_iter().map(|(key, value)| { + account_storage_map_details::map_entries::StorageMapEntry { + key: Some(key.into()), + value: Some(value.into()), + } + })), + }); + + Self { + slot_name: slot_name.to_string(), + too_many_entries: false, + entries, } - })), - }); - - Self { - slot_name: slot_name.to_string(), - too_many_entries, - entries, + }, } } } diff --git a/crates/store/src/db/migrations/2025062000000_setup/up.sql b/crates/store/src/db/migrations/2025062000000_setup/up.sql index aaafb91a81..3f7449292d 100644 --- a/crates/store/src/db/migrations/2025062000000_setup/up.sql +++ b/crates/store/src/db/migrations/2025062000000_setup/up.sql @@ -18,17 +18,17 @@ CREATE TABLE accounts ( block_num INTEGER NOT NULL, account_commitment BLOB NOT NULL, code_commitment BLOB, - storage BLOB, - vault BLOB, nonce INTEGER, + storage_header BLOB, -- Serialized AccountStorage from miden-objects + vault_root BLOB, -- Vault root commitment is_latest BOOLEAN NOT NULL DEFAULT 0, -- Indicates if this is the latest state for this account_id PRIMARY KEY (account_id, block_num), CONSTRAINT all_null_or_none_null CHECK ( - (code_commitment IS NOT NULL AND storage IS NOT NULL AND vault IS NOT NULL AND nonce IS NOT NULL) + (code_commitment IS NOT NULL AND nonce IS NOT NULL AND storage_header IS NOT NULL AND vault_root IS NOT NULL) OR - (code_commitment IS NULL AND storage IS NULL AND vault IS NULL AND nonce IS NULL) + (code_commitment IS NULL AND nonce IS NULL AND storage_header IS NULL AND vault_root IS NULL) ) ) WITHOUT ROWID; diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 9083089f31..9f88f0090a 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -7,7 +7,7 @@ use diesel::{Connection, RunQueryDsl, SqliteConnection}; use miden_node_proto::domain::account::{AccountInfo, AccountSummary, NetworkAccountPrefix}; use miden_node_proto::generated as proto; use miden_protocol::Word; -use miden_protocol::account::AccountId; +use miden_protocol::account::{AccountHeader, AccountId, AccountStorage}; use miden_protocol::asset::{Asset, AssetVaultKey}; use miden_protocol::block::{BlockHeader, BlockNoteIndex, BlockNumber, ProvenBlock}; use miden_protocol::crypto::merkle::SparseMerklePath; @@ -112,8 +112,7 @@ impl TransactionRecord { self, note_records: Vec, ) -> proto::rpc::TransactionRecord { - let output_notes: Vec = - note_records.into_iter().map(Into::into).collect(); + let output_notes = Vec::from_iter(note_records.into_iter().map(Into::into)); proto::rpc::TransactionRecord { header: Some(proto::transaction::TransactionHeader { @@ -323,7 +322,7 @@ impl Db { /// Loads all the nullifiers from the DB. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub async fn select_all_nullifiers(&self) -> Result> { + pub(crate) async fn select_all_nullifiers(&self) -> Result> { self.transact("all nullifiers", move |conn| { let nullifiers = queries::select_all_nullifiers(conn)?; Ok(nullifiers) @@ -392,7 +391,7 @@ impl Db { .await } - /// Loads all the account commitments from the DB. + /// TODO marked for removal, replace with paged version #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_account_commitments(&self) -> Result> { self.transact("read all account commitments", move |conn| { @@ -401,6 +400,16 @@ impl Db { .await } + /// Returns all account IDs that have public state. + #[allow(dead_code)] // Will be used by InnerForest in next PR + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] + pub async fn select_all_public_account_ids(&self) -> Result> { + self.transact("read all public account IDs", move |conn| { + queries::select_all_public_account_ids(conn) + }) + .await + } + /// Loads public account details from the DB. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_account(&self, id: AccountId) -> Result { @@ -408,19 +417,6 @@ impl Db { .await } - /// Loads account details at a specific block number from the DB. - #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub async fn select_historical_account_at( - &self, - id: AccountId, - block_num: BlockNumber, - ) -> Result { - self.transact("Get historical account details", move |conn| { - queries::select_historical_account_at(conn, id, block_num) - }) - .await - } - /// Loads public account details from the DB based on the account ID's prefix. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_network_account_by_prefix( @@ -440,6 +436,64 @@ impl Db { .await } + /// Reconstructs account storage at a specific block from the database + /// + /// This method queries the decomposed storage tables and reconstructs the full + /// `AccountStorage` with SMT backing for Map slots. + // TODO split querying the header from the content + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] + pub async fn select_account_storage_at_block( + &self, + account_id: AccountId, + block_num: BlockNumber, + ) -> Result { + self.transact("Get account storage at block", move |conn| { + queries::select_account_storage_at_block(conn, account_id, block_num) + }) + .await + } + + /// Queries vault assets at a specific block + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] + pub async fn select_account_vault_at_block( + &self, + account_id: AccountId, + block_num: BlockNumber, + ) -> Result> { + self.transact("Get account vault at block", move |conn| { + queries::select_account_vault_at_block(conn, account_id, block_num) + }) + .await + } + + /// Queries the account code for a specific account at a specific block number. + /// + /// Returns `None` if the account doesn't exist at that block or has no code. + pub async fn select_account_code_at_block( + &self, + account_id: AccountId, + block_num: BlockNumber, + ) -> Result>> { + self.transact("Get account code at block", move |conn| { + queries::select_account_code_at_block(conn, account_id, block_num) + }) + .await + } + + /// Queries the account header for a specific account at a specific block number. + /// + /// Returns `None` if the account doesn't exist at that block. + pub async fn select_account_header_at_block( + &self, + account_id: AccountId, + block_num: BlockNumber, + ) -> Result> { + self.transact("Get account header at block", move |conn| { + queries::select_account_header_at_block(conn, account_id, block_num) + }) + .await + } + #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn get_state_sync( &self, @@ -538,7 +592,7 @@ impl Db { .await } - /// Selects storage map values for syncing storage maps for a specific account ID. + /// Selects storage map values for syncing storage maps for a specific account ID /// /// The returned values are the latest known values up to `block_range.end()`, and no values /// earlier than `block_range.start()` are returned. diff --git a/crates/store/src/db/models/conv.rs b/crates/store/src/db/models/conv.rs index 48013b370d..37a9b019fb 100644 --- a/crates/store/src/db/models/conv.rs +++ b/crates/store/src/db/models/conv.rs @@ -34,7 +34,7 @@ use miden_node_proto::domain::account::NetworkAccountPrefix; use miden_protocol::Felt; -use miden_protocol::account::StorageSlotName; +use miden_protocol::account::{StorageSlotName, StorageSlotType}; use miden_protocol::block::BlockNumber; use miden_protocol::note::{NoteExecutionMode, NoteTag}; @@ -131,6 +131,33 @@ impl SqlTypeConvert for NoteTag { } } +impl SqlTypeConvert for StorageSlotType { + type Raw = i32; + + #[inline(always)] + fn from_raw_sql(raw: Self::Raw) -> Result { + #[derive(Debug, thiserror::Error)] + #[error("invalid storage slot type value {0}")] + struct ValueError(i32); + + Ok(match raw { + 0 => StorageSlotType::Value, + 1 => StorageSlotType::Map, + invalid => { + return Err(Self::map_err(ValueError(invalid))); + }, + }) + } + + #[inline(always)] + fn to_raw_sql(self) -> Self::Raw { + match self { + StorageSlotType::Value => 0, + StorageSlotType::Map => 1, + } + } +} + impl SqlTypeConvert for StorageSlotName { type Raw = String; @@ -157,9 +184,9 @@ pub(crate) fn nullifier_prefix_to_raw_sql(prefix: u16) -> i32 { } #[inline(always)] -pub(crate) fn raw_sql_to_nonce(raw: i64) -> u64 { +pub(crate) fn raw_sql_to_nonce(raw: i64) -> Felt { debug_assert!(raw >= 0); - raw as u64 + Felt::new(raw as u64) } #[inline(always)] pub(crate) fn nonce_to_raw_sql(nonce: Felt) -> i64 { diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 913adfc405..290b5d7499 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::ops::RangeInclusive; use diesel::prelude::{Queryable, QueryableByName}; @@ -8,8 +9,6 @@ use diesel::{ BoolExpressionMethods, ExpressionMethods, Insertable, - JoinOnDsl, - NullableExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, @@ -24,6 +23,7 @@ use miden_node_utils::limiter::{ QueryParamAccountIdLimit, QueryParamLimiter, }; +use miden_protocol::Word; use miden_protocol::account::delta::AccountUpdateDetails; use miden_protocol::account::{ Account, @@ -31,28 +31,41 @@ use miden_protocol::account::{ AccountDelta, AccountId, AccountStorage, + AccountStorageHeader, NonFungibleDeltaAction, + StorageMap, + StorageSlot, StorageSlotContent, StorageSlotName, + StorageSlotType, }; use miden_protocol::asset::{Asset, AssetVault, AssetVaultKey, FungibleAsset}; use miden_protocol::block::{BlockAccountUpdate, BlockNumber}; use miden_protocol::utils::{Deserializable, Serializable}; -use miden_protocol::{Felt, Word}; use crate::db::models::conv::{SqlTypeConvert, nonce_to_raw_sql, raw_sql_to_nonce}; use crate::db::models::{serialize_vec, vec_raw_try_into}; use crate::db::{AccountVaultValue, schema}; use crate::errors::DatabaseError; +mod at_block; +pub(crate) use at_block::{ + select_account_code_at_block, + select_account_header_at_block, + select_account_storage_at_block, + select_account_vault_at_block, +}; + type StorageMapValueRow = (i64, String, Vec, Vec); -/// Select the latest account details by account id from the DB using the given -/// [`SqliteConnection`]. +// ACCOUNT RETRIEVAL +// ================================================================================================ + +/// Select account by ID from the DB using the given [`SqliteConnection`]. /// /// # Returns /// -/// The latest account details, or an error. +/// The latest account info, or an error. /// /// # Raw SQL /// @@ -60,16 +73,9 @@ type StorageMapValueRow = (i64, String, Vec, Vec); /// SELECT /// accounts.account_id, /// accounts.account_commitment, -/// accounts.block_num, -/// accounts.storage, -/// accounts.vault, -/// accounts.nonce, -/// accounts.code_commitment, -/// account_codes.code +/// accounts.block_num /// FROM /// accounts -/// LEFT JOIN -/// account_codes ON accounts.code_commitment = account_codes.code_commitment /// WHERE /// account_id = ?1 /// AND is_latest = 1 @@ -78,85 +84,96 @@ pub(crate) fn select_account( conn: &mut SqliteConnection, account_id: AccountId, ) -> Result { - let raw = SelectDsl::select( - schema::accounts::table.left_join(schema::account_codes::table.on( - schema::accounts::code_commitment.eq(schema::account_codes::code_commitment.nullable()), - )), - (AccountRaw::as_select(), schema::account_codes::code.nullable()), - ) - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::is_latest.eq(true)) - .get_result::<(AccountRaw, Option>)>(conn) - .optional()? - .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; - let info = AccountWithCodeRawJoined::from(raw).try_into()?; - Ok(info) + let raw = SelectDsl::select(schema::accounts::table, AccountSummaryRaw::as_select()) + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .get_result::(conn) + .optional()? + .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + + let summary: AccountSummary = raw.try_into()?; + + // Backfill account details from database + // For private accounts, we don't store full details in the database + let details = if account_id.has_public_state() { + Some(select_full_account(conn, account_id)?) + } else { + None + }; + + Ok(AccountInfo { summary, details }) } -/// Select account details as they are at the given block height. -/// -/// # Returns +/// Reconstruct full Account from database tables for the latest account state /// -/// The account details at the specified block, or an error. +/// This function queries the database tables to reconstruct a complete Account object: +/// - Code from `account_codes` table +/// - Nonce and storage header from `accounts` table +/// - Storage map entries from `account_storage_map_values` table +/// - Vault from `account_vault_assets` table /// -/// # Raw SQL +/// # Note /// -/// ```sql -/// SELECT -/// accounts.account_id, -/// accounts.account_commitment, -/// accounts.block_num, -/// accounts.storage, -/// accounts.vault, -/// accounts.nonce, -/// accounts.code_commitment, -/// account_codes.code -/// FROM -/// accounts -/// LEFT JOIN -/// account_codes ON accounts.code_commitment = account_codes.code_commitment -/// WHERE -/// account_id = ?1 -/// AND block_num <= ?2 -/// ORDER BY -/// block_num DESC -/// LIMIT -/// 1 -/// ``` -pub(crate) fn select_historical_account_at( +/// A stop-gap solution to retain store API and construct `AccountInfo` types. +/// The function should ultimately be removed, and any queries be served from the +/// `State` which contains an `SmtForest` to serve the latest and most recent +/// historical data. +// TODO: remove eventually once refactoring is complete +fn select_full_account( conn: &mut SqliteConnection, account_id: AccountId, - block_num: BlockNumber, -) -> Result { - let raw = SelectDsl::select( - schema::accounts::table.left_join(schema::account_codes::table.on( - schema::accounts::code_commitment.eq(schema::account_codes::code_commitment.nullable()), - )), - (AccountRaw::as_select(), schema::account_codes::code.nullable()), +) -> Result { + // Get account metadata (nonce, code_commitment) and code in a single join query + let (nonce, code_bytes): (Option, Vec) = SelectDsl::select( + schema::accounts::table.inner_join(schema::account_codes::table), + (schema::accounts::nonce, schema::account_codes::code), ) - .filter( - schema::accounts::account_id - .eq(account_id.to_bytes()) - .and(schema::accounts::block_num.le(block_num.to_raw_sql())), - ) - .order_by(schema::accounts::block_num.desc()) - .limit(1) - .get_result::<(AccountRaw, Option>)>(conn) + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .get_result(conn) .optional()? .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; - let info = AccountWithCodeRawJoined::from(raw).try_into()?; - Ok(info) + + let nonce = raw_sql_to_nonce(nonce.ok_or_else(|| { + DatabaseError::DataCorrupted(format!("No nonce found for account {account_id}")) + })?); + + let code = AccountCode::read_from_bytes(&code_bytes)?; + + // Reconstruct storage using existing helper function + let storage = select_latest_account_storage(conn, account_id)?; + + // Reconstruct vault from account_vault_assets table + let vault_entries: Vec<(Vec, Option>)> = SelectDsl::select( + schema::account_vault_assets::table, + (schema::account_vault_assets::vault_key, schema::account_vault_assets::asset), + ) + .filter(schema::account_vault_assets::account_id.eq(account_id.to_bytes())) + .filter(schema::account_vault_assets::is_latest.eq(true)) + .load(conn)?; + + let mut assets = Vec::new(); + for (_key_bytes, maybe_asset_bytes) in vault_entries { + if let Some(asset_bytes) = maybe_asset_bytes { + let asset = Asset::read_from_bytes(&asset_bytes)?; + assets.push(asset); + } + } + + let vault = AssetVault::new(&assets)?; + + Ok(Account::new(account_id, vault, storage, code, nonce, None)?) } -/// Select the latest account details by account ID prefix from the DB using the given -/// [`SqliteConnection`] This method is meant to be used by the network transaction builder. Because -/// network notes get matched through accounts through the account's 30-bit prefix, it is possible -/// that multiple accounts match against a single prefix. In this scenario, the first account is -/// returned. +/// Select the latest account info by account ID prefix from the DB using the given +/// [`SqliteConnection`]. Meant to be used by the network transaction builder. +/// Because network notes get matched through accounts through the account's 30-bit prefix, it is +/// possible that multiple accounts match against a single prefix. In this scenario, the first +/// account is returned. /// /// # Returns /// -/// The latest account details, `None` if the account was not found, or an error. +/// The latest account info, `None` if the account was not found, or an error. /// /// # Raw SQL /// @@ -164,41 +181,34 @@ pub(crate) fn select_historical_account_at( /// SELECT /// accounts.account_id, /// accounts.account_commitment, -/// accounts.block_num, -/// accounts.storage, -/// accounts.vault, -/// accounts.nonce, -/// accounts.code_commitment, -/// account_codes.code +/// accounts.block_num /// FROM /// accounts -/// LEFT JOIN -/// account_codes ON accounts.code_commitment = account_codes.code_commitment /// WHERE /// network_account_id_prefix = ?1 +/// AND is_latest = 1 /// ``` pub(crate) fn select_account_by_id_prefix( conn: &mut SqliteConnection, id_prefix: u32, ) -> Result, DatabaseError> { - let maybe_info = SelectDsl::select( - schema::accounts::table.left_join(schema::account_codes::table.on( - schema::accounts::code_commitment.eq(schema::account_codes::code_commitment.nullable()), - )), - (AccountRaw::as_select(), schema::account_codes::code.nullable()), - ) - .filter(schema::accounts::is_latest.eq(true)) - .filter(schema::accounts::network_account_id_prefix.eq(Some(i64::from(id_prefix)))) - .get_result::<(AccountRaw, Option>)>(conn) - .optional() - .map_err(DatabaseError::Diesel)?; - - let result: Result, DatabaseError> = maybe_info - .map(AccountWithCodeRawJoined::from) - .map(std::convert::TryInto::::try_into) - .transpose(); - - result + let maybe_summary = SelectDsl::select(schema::accounts::table, AccountSummaryRaw::as_select()) + .filter(schema::accounts::is_latest.eq(true)) + .filter(schema::accounts::network_account_id_prefix.eq(Some(i64::from(id_prefix)))) + .get_result::(conn) + .optional() + .map_err(DatabaseError::Diesel)?; + + match maybe_summary { + None => Ok(None), + Some(raw) => { + let summary: AccountSummary = raw.try_into()?; + let account_id = summary.account_id; + // Backfill account details from database + let details = select_full_account(conn, account_id).ok(); + Ok(Some(AccountInfo { summary, details })) + }, + } } /// Select all account commitments from the DB using the given [`SqliteConnection`]. @@ -238,6 +248,48 @@ pub(crate) fn select_all_account_commitments( )) } +/// Select all account IDs that have public state. +/// +/// This filters accounts in-memory after loading only the account IDs (not commitments), +/// which is more efficient than loading full commitments when only IDs are needed. +/// +/// # Raw SQL +/// +/// ```sql +/// SELECT +/// account_id +/// FROM +/// accounts +/// WHERE +/// is_latest = 1 +/// ORDER BY +/// block_num ASC +/// ``` +#[allow(dead_code)] // Will be used by InnerForest in next PR +pub(crate) fn select_all_public_account_ids( + conn: &mut SqliteConnection, +) -> Result, DatabaseError> { + // We could technically use a `LIKE` constraint for both postgres and sqlite backends, + // but diesel doesn't expose that. + let raw: Vec> = + SelectDsl::select(schema::accounts::table, schema::accounts::account_id) + .filter(schema::accounts::is_latest.eq(true)) + .order_by(schema::accounts::block_num.asc()) + .load::>(conn)?; + + Result::from_iter( + raw.into_iter() + .map(|bytes| { + AccountId::read_from_bytes(&bytes).map_err(DatabaseError::DeserializationError) + }) + .filter_map(|result| match result { + Ok(id) if id.has_public_state() => Some(Ok(id)), + Ok(_) => None, + Err(e) => Some(Err(e)), + }), + ) +} + /// Select account vault assets within a block range (inclusive). /// /// # Parameters @@ -379,16 +431,11 @@ pub fn select_accounts_by_block_range( /// SELECT /// accounts.account_id, /// accounts.account_commitment, -/// accounts.block_num, -/// accounts.storage, -/// accounts.vault, -/// accounts.nonce, -/// accounts.code_commitment, -/// account_codes.code +/// accounts.block_num /// FROM /// accounts -/// LEFT JOIN -/// account_codes ON accounts.code_commitment = account_codes.code_commitment +/// WHERE +/// is_latest = 1 /// ORDER BY /// block_num ASC /// ``` @@ -396,17 +443,20 @@ pub fn select_accounts_by_block_range( pub(crate) fn select_all_accounts( conn: &mut SqliteConnection, ) -> Result, DatabaseError> { - let accounts_raw = QueryDsl::select( - schema::accounts::table.left_join(schema::account_codes::table.on( - schema::accounts::code_commitment.eq(schema::account_codes::code_commitment.nullable()), - )), - (AccountRaw::as_select(), schema::account_codes::code.nullable()), - ) - .filter(schema::accounts::is_latest.eq(true)) - .load::<(AccountRaw, Option>)>(conn)?; - let account_infos = vec_raw_try_into::( - accounts_raw.into_iter().map(AccountWithCodeRawJoined::from), - )?; + let raw = SelectDsl::select(schema::accounts::table, AccountSummaryRaw::as_select()) + .filter(schema::accounts::is_latest.eq(true)) + .order_by(schema::accounts::block_num.asc()) + .load::(conn)?; + + let summaries: Vec = vec_raw_try_into(raw).unwrap(); + + // Backfill account details from database + let account_infos = Vec::from_iter(summaries.into_iter().map(|summary| { + let account_id = summary.account_id; + let details = select_full_account(conn, account_id).ok(); + AccountInfo { summary, details } + })); + Ok(account_infos) } @@ -565,6 +615,76 @@ pub(crate) fn select_account_storage_map_values( Ok(StorageMapValuesPage { last_block_included, values }) } +/// Select latest account storage by querying `accounts.storage_header` where `is_latest=true` +/// and reconstructing full storage from the header plus map values from +/// `account_storage_map_values`. +pub(crate) fn select_latest_account_storage( + conn: &mut SqliteConnection, + account_id: AccountId, +) -> Result { + use schema::account_storage_map_values as t; + + let account_id_bytes = account_id.to_bytes(); + + // Query storage header blob for this account where is_latest = true + let storage_blob: Option> = + SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) + .filter(schema::accounts::account_id.eq(&account_id_bytes)) + .filter(schema::accounts::is_latest.eq(true)) + .first(conn) + .optional()? + .flatten(); + + let Some(blob) = storage_blob else { + // No storage means empty storage + return Ok(AccountStorage::new(Vec::new())?); + }; + + // Deserialize the AccountStorageHeader from the blob + let header = AccountStorageHeader::read_from_bytes(&blob)?; + + // Query all latest map values for this account + let map_values: Vec<(String, Vec, Vec)> = + SelectDsl::select(t::table, (t::slot_name, t::key, t::value)) + .filter(t::account_id.eq(&account_id_bytes)) + .filter(t::is_latest.eq(true)) + .load(conn)?; + + // Group map values by slot name + let mut map_entries_by_slot: BTreeMap> = BTreeMap::new(); + for (slot_name_str, key_bytes, value_bytes) in map_values { + let slot_name: StorageSlotName = slot_name_str.parse().map_err(|_| { + DatabaseError::DataCorrupted(format!("Invalid slot name: {slot_name_str}")) + })?; + let key = Word::read_from_bytes(&key_bytes)?; + let value = Word::read_from_bytes(&value_bytes)?; + map_entries_by_slot.entry(slot_name).or_default().push((key, value)); + } + + // Reconstruct StorageSlots from header slots + map entries + let mut slots = Vec::new(); + for slot_header in header.slots() { + let slot = match slot_header.slot_type() { + StorageSlotType::Value => { + // For value slots, the header value IS the slot value + StorageSlot::with_value(slot_header.name().clone(), slot_header.value()) + }, + StorageSlotType::Map => { + // For map slots, reconstruct from map entries + let entries = map_entries_by_slot.remove(slot_header.name()).unwrap_or_default(); + let storage_map = StorageMap::with_entries(entries)?; + StorageSlot::with_map(slot_header.name().clone(), storage_map) + }, + }; + slots.push(slot); + } + + Ok(AccountStorage::new(slots)?) +} + +// ACCOUNT MUTATION +// ================================================================================================ + #[derive(Queryable, Selectable)] #[diesel(table_name = crate::db::schema::account_vault_assets)] #[diesel(check_for_backend(diesel::sqlite::Sqlite))] @@ -586,73 +706,6 @@ impl TryFrom for AccountVaultValue { } } -#[derive(Debug, Clone, Queryable, QueryableByName, Selectable)] -#[diesel(table_name = schema::accounts)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -pub struct AccountRaw { - pub account_id: Vec, - pub account_commitment: Vec, - pub block_num: i64, - pub storage: Option>, - pub vault: Option>, - pub nonce: Option, -} - -#[derive(Debug, Clone, QueryableByName)] -pub struct AccountWithCodeRawJoined { - #[diesel(embed)] - pub account: AccountRaw, - #[diesel(embed)] - pub code: Option>, -} - -impl From<(AccountRaw, Option>)> for AccountWithCodeRawJoined { - fn from((account, code): (AccountRaw, Option>)) -> Self { - Self { account, code } - } -} - -impl TryInto for AccountWithCodeRawJoined { - type Error = DatabaseError; - fn try_into(self) -> Result { - use proto::domain::account::{AccountInfo, AccountSummary}; - - let account_id = AccountId::read_from_bytes(&self.account.account_id[..])?; - let account_commitment = Word::read_from_bytes(&self.account.account_commitment[..])?; - let block_num = BlockNumber::from_raw_sql(self.account.block_num)?; - let summary = AccountSummary { - account_id, - account_commitment, - block_num, - }; - let maybe_account = self.try_into()?; - Ok(AccountInfo { summary, details: maybe_account }) - } -} - -impl TryInto> for AccountWithCodeRawJoined { - type Error = DatabaseError; - fn try_into(self) -> Result, Self::Error> { - let account_id = AccountId::read_from_bytes(&self.account.account_id[..])?; - - let details = if let (Some(vault), Some(storage), Some(nonce), Some(code)) = - (self.account.vault, self.account.storage, self.account.nonce, self.code) - { - let vault = AssetVault::read_from_bytes(&vault)?; - let storage = AccountStorage::read_from_bytes(&storage)?; - let code = AccountCode::read_from_bytes(&code)?; - let nonce = raw_sql_to_nonce(nonce); - let nonce = Felt::new(nonce); - let account = Account::new_unchecked(account_id, vault, storage, code, nonce, None); - Some(account) - } else { - // a private account - None - }; - Ok(details) - } -} - #[derive(Debug, Clone, PartialEq, Eq, Selectable, Queryable, QueryableByName)] #[diesel(table_name = schema::accounts)] #[diesel(check_for_backend(Sqlite))] @@ -679,7 +732,7 @@ impl TryInto for AccountSummaryRaw { /// Insert an account vault asset row into the DB using the given [`SqliteConnection`]. /// -/// This function will set `is_latest=true` for the new row and update any existing +/// Sets `is_latest=true` for the new row and updates any existing /// row with the same `(account_id, vault_key)` tuple to `is_latest=false`. /// /// # Returns @@ -719,8 +772,8 @@ pub(crate) fn insert_account_vault_asset( /// Insert an account storage map value into the DB using the given [`SqliteConnection`]. /// -/// This function will set `is_latest=true` for the new row and update any existing -/// row with the same `(account_id, slot, key)` tuple to `is_latest=false`. +/// Sets `is_latest=true` for the new row and updates any existing +/// row with the same `(account_id, slot_index, key)` tuple to `is_latest=false`. /// /// # Returns /// @@ -774,32 +827,6 @@ pub(crate) fn upsert_accounts( ) -> Result { use proto::domain::account::NetworkAccountPrefix; - fn select_details_stmt( - conn: &mut SqliteConnection, - account_id: AccountId, - ) -> Result, DatabaseError> { - let account_id = account_id.to_bytes(); - let accounts = SelectDsl::select( - schema::accounts::table.left_join( - schema::account_codes::table.on(schema::accounts::code_commitment - .eq(schema::account_codes::code_commitment.nullable())), - ), - (AccountRaw::as_select(), schema::account_codes::code.nullable()), - ) - .filter(schema::accounts::account_id.eq(account_id)) - .filter(schema::accounts::is_latest.eq(true)) - .get_results::<(AccountRaw, Option>)>(conn)?; - - // SELECT .. FROM accounts LEFT JOIN account_codes - // ON accounts.code_commitment == account_codes.code_commitment - - let accounts = Result::from_iter(accounts.into_iter().filter_map(|x| { - let account_with_code = AccountWithCodeRawJoined::from(x); - account_with_code.try_into().transpose() - }))?; - Ok(accounts) - } - let mut count = 0; for update in accounts { let account_id = update.account_id(); @@ -856,10 +883,8 @@ pub(crate) fn upsert_accounts( }, AccountUpdateDetails::Delta(delta) => { - let mut rows = select_details_stmt(conn, account_id)?.into_iter(); - let Some(account_before) = rows.next() else { - return Err(DatabaseError::AccountNotFoundInDb(account_id)); - }; + // Reconstruct the full account from database tables + let account = select_full_account(conn, account_id)?; // --- collect storage map updates ---------------------------- @@ -873,8 +898,7 @@ pub(crate) fn upsert_accounts( // apply delta to the account; we need to do this before we process asset updates // because we currently need to get the current value of fungible assets from the // account - let account_after = - apply_delta(account_before, delta, &update.final_state_commitment())?; + let account_after = apply_delta(account, delta, &update.final_state_commitment())?; // --- process asset updates ---------------------------------- @@ -929,11 +953,14 @@ pub(crate) fn upsert_accounts( account_commitment: update.final_state_commitment().to_bytes(), block_num: block_num.to_raw_sql(), nonce: full_account.as_ref().map(|account| nonce_to_raw_sql(account.nonce())), - storage: full_account.as_ref().map(|account| account.storage().to_bytes()), - vault: full_account.as_ref().map(|account| account.vault().to_bytes()), code_commitment: full_account .as_ref() .map(|account| account.code().commitment().to_bytes()), + // Store only the header (slot metadata + map roots), not full storage with map contents + storage_header: full_account + .as_ref() + .map(|account| account.storage().to_header().to_bytes()), + vault_root: full_account.as_ref().map(|account| account.vault().root().to_bytes()), is_latest: true, }; @@ -946,7 +973,6 @@ pub(crate) fn upsert_accounts( insert_account_storage_map_value(conn, acc_id, block_num, slot_name, key, value)?; } - // insert pending vault-asset entries for (acc_id, vault_key, update) in pending_asset_inserts { insert_account_vault_asset(conn, acc_id, block_num, vault_key, update)?; } @@ -991,9 +1017,9 @@ pub(crate) struct AccountRowInsert { pub(crate) block_num: i64, pub(crate) account_commitment: Vec, pub(crate) code_commitment: Option>, - pub(crate) storage: Option>, - pub(crate) vault: Option>, pub(crate) nonce: Option, + pub(crate) storage_header: Option>, + pub(crate) vault_root: Option>, pub(crate) is_latest: bool, } diff --git a/crates/store/src/db/models/queries/accounts/at_block.rs b/crates/store/src/db/models/queries/accounts/at_block.rs new file mode 100644 index 0000000000..aaef34a159 --- /dev/null +++ b/crates/store/src/db/models/queries/accounts/at_block.rs @@ -0,0 +1,315 @@ +use std::collections::BTreeMap; + +use diesel::prelude::Queryable; +use diesel::query_dsl::methods::SelectDsl; +use diesel::{ + BoolExpressionMethods, + ExpressionMethods, + OptionalExtension, + QueryDsl, + RunQueryDsl, + SqliteConnection, +}; +use miden_protocol::account::{ + AccountHeader, + AccountId, + AccountStorage, + AccountStorageHeader, + StorageMap, + StorageSlot, + StorageSlotName, + StorageSlotType, +}; +use miden_protocol::asset::Asset; +use miden_protocol::block::BlockNumber; +use miden_protocol::utils::{Deserializable, Serializable}; +use miden_protocol::{Felt, FieldElement, Word}; + +use crate::db::models::conv::{SqlTypeConvert, raw_sql_to_nonce}; +use crate::db::schema; +use crate::errors::DatabaseError; + +// ACCOUNT HEADER +// ================================================================================================ + +#[derive(Debug, Clone, Queryable)] +struct AccountHeaderDataRaw { + code_commitment: Option>, + nonce: Option, + storage_header: Option>, +} + +/// Queries the account header for a specific account at a specific block number. +/// +/// This reconstructs the `AccountHeader` by reading from the `accounts` table: +/// - `account_id`, `nonce`, `code_commitment`, `storage_header`, `vault_root` +/// +/// Returns `None` if the account doesn't exist at that block. +/// +/// # Arguments +/// +/// * `conn` - Database connection +/// * `account_id` - The account ID to query +/// * `block_num` - The block number at which to query the account header +/// +/// # Returns +/// +/// * `Ok(Some(AccountHeader))` - The account header if found +/// * `Ok(None)` - If account doesn't exist at that block +/// * `Err(DatabaseError)` - If there's a database error +pub(crate) fn select_account_header_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { + use schema::accounts; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + + let account_data: Option<(AccountHeaderDataRaw, Option>)> = SelectDsl::select( + accounts::table + .filter(accounts::account_id.eq(&account_id_bytes)) + .filter(accounts::block_num.le(block_num_sql)) + .order(accounts::block_num.desc()) + .limit(1), + ( + (accounts::code_commitment, accounts::nonce, accounts::storage_header), + accounts::vault_root, + ), + ) + .first(conn) + .optional()?; + + let Some(( + AccountHeaderDataRaw { + code_commitment: code_commitment_bytes, + nonce: nonce_raw, + storage_header: storage_header_blob, + }, + vault_root_bytes, + )) = account_data + else { + return Ok(None); + }; + + let storage_commitment = match storage_header_blob { + Some(blob) => { + let header = AccountStorageHeader::read_from_bytes(&blob)?; + header.to_commitment() + }, + None => Word::default(), + }; + + let code_commitment = code_commitment_bytes + .map(|bytes| Word::read_from_bytes(&bytes)) + .transpose()? + .unwrap_or(Word::default()); + + let nonce = nonce_raw.map_or(Felt::ZERO, raw_sql_to_nonce); + + let vault_root = vault_root_bytes + .map(|bytes| Word::read_from_bytes(&bytes)) + .transpose()? + .unwrap_or(Word::default()); + + Ok(Some(AccountHeader::new( + account_id, + nonce, + vault_root, + storage_commitment, + code_commitment, + ))) +} + +// ACCOUNT CODE +// ================================================================================================ + +/// Queries the account code for a specific account at a specific block number. +/// +/// Returns `None` if: +/// - The account doesn't exist at that block +/// - The account has no code (private account or account without code commitment) +/// +/// # Arguments +/// +/// * `conn` - Database connection +/// * `account_id` - The account ID to query +/// * `block_num` - The block number at which to query the account code +/// +/// # Returns +/// +/// * `Ok(Some(Vec))` - The account code bytes if found +/// * `Ok(None)` - If account doesn't exist or has no code +/// * `Err(DatabaseError)` - If there's a database error +pub(crate) fn select_account_code_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result>, DatabaseError> { + use schema::{account_codes, accounts}; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = i64::from(block_num.as_u32()); + // Query the accounts table to get the code_commitment at the specified block or earlier + // Then join with account_codes to get the actual code + let result: Option> = SelectDsl::select( + accounts::table + .inner_join(account_codes::table) + .filter(accounts::account_id.eq(&account_id_bytes)) + .filter(accounts::block_num.le(block_num_sql)) + .order(accounts::block_num.desc()) + .limit(1), + account_codes::code, + ) + .first(conn) + .optional()?; + + Ok(result) +} + +// ACCOUNT VAULT +// ================================================================================================ + +/// Query vault assets at a specific block by finding the most recent update for each `vault_key`. +pub(crate) fn select_account_vault_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { + use schema::account_vault_assets as t; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + + // Since Diesel doesn't support composite keys in subqueries easily, we use a two-step approach: + // Step 1: Get max block_num for each vault_key + let latest_blocks_per_vault_key = Vec::from_iter( + QueryDsl::select( + t::table + .filter(t::account_id.eq(&account_id_bytes)) + .filter(t::block_num.le(block_num_sql)) + .group_by(t::vault_key), + (t::vault_key, diesel::dsl::max(t::block_num)), + ) + .load::<(Vec, Option)>(conn)? + .into_iter() + .filter_map(|(key, maybe_block)| maybe_block.map(|block| (key, block))), + ); + + if latest_blocks_per_vault_key.is_empty() { + return Ok(Vec::new()); + } + + // Step 2: Fetch the full rows matching (vault_key, block_num) pairs + let mut assets = Vec::new(); + for (vault_key_bytes, max_block) in latest_blocks_per_vault_key { + let result: Option>> = QueryDsl::select( + t::table.filter( + t::account_id + .eq(&account_id_bytes) + .and(t::vault_key.eq(&vault_key_bytes)) + .and(t::block_num.eq(max_block)), + ), + t::asset, + ) + .first(conn) + .optional()?; + if let Some(Some(asset_bytes)) = result { + let asset = Asset::read_from_bytes(&asset_bytes)?; + assets.push(asset); + } + } + + // Sort by vault_key for consistent ordering + assets.sort_by_key(Asset::vault_key); + + Ok(assets) +} + +// ACCOUNT STORAGE +// ================================================================================================ + +/// Returns account storage at a given block by reading from `accounts.storage_header` +/// (which contains the `AccountStorageHeader`) and reconstructing full storage from +/// map values in `account_storage_map_values` table. +pub(crate) fn select_account_storage_at_block( + conn: &mut SqliteConnection, + account_id: AccountId, + block_num: BlockNumber, +) -> Result { + use schema::account_storage_map_values as t; + + let account_id_bytes = account_id.to_bytes(); + let block_num_sql = block_num.to_raw_sql(); + + // Query storage header blob for this account at or before this block + let storage_blob: Option> = + SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) + .filter(schema::accounts::account_id.eq(&account_id_bytes)) + .filter(schema::accounts::block_num.le(block_num_sql)) + .order(schema::accounts::block_num.desc()) + .limit(1) + .first(conn) + .optional()? + .flatten(); + + let Some(blob) = storage_blob else { + // No storage means empty storage + return Ok(AccountStorage::new(Vec::new())?); + }; + + // Deserialize the AccountStorageHeader from the blob + let header = AccountStorageHeader::read_from_bytes(&blob)?; + + // Query all map values for this account up to and including this block. + // For each (slot_name, key), we need the latest value at or before block_num. + // First, get all entries up to block_num + let map_values: Vec<(i64, String, Vec, Vec)> = + SelectDsl::select(t::table, (t::block_num, t::slot_name, t::key, t::value)) + .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.le(block_num_sql))) + .order((t::slot_name.asc(), t::key.asc(), t::block_num.desc())) + .load(conn)?; + + // For each (slot_name, key) pair, keep only the latest entry (highest block_num) + let mut latest_map_entries: BTreeMap<(StorageSlotName, Word), Word> = BTreeMap::new(); + + for (_, slot_name_str, key_bytes, value_bytes) in map_values { + let slot_name: StorageSlotName = slot_name_str.parse().map_err(|_| { + DatabaseError::DataCorrupted(format!("Invalid slot name: {slot_name_str}")) + })?; + let key = Word::read_from_bytes(&key_bytes)?; + + // Only insert if we haven't seen this (slot_name, key) yet + // (since results are ordered by block_num desc, first one is latest) + latest_map_entries + .entry((slot_name, key)) + .or_insert_with(|| Word::read_from_bytes(&value_bytes).unwrap_or_default()); + } + + // Group entries by slot name + let mut map_entries_by_slot: BTreeMap> = BTreeMap::new(); + for ((slot_name, key), value) in latest_map_entries { + map_entries_by_slot.entry(slot_name).or_default().push((key, value)); + } + + // Reconstruct StorageSlots from header slots + map entries + let mut slots = Vec::new(); + for slot_header in header.slots() { + let slot = match slot_header.slot_type() { + StorageSlotType::Value => { + // For value slots, the header value IS the slot value + StorageSlot::with_value(slot_header.name().clone(), slot_header.value()) + }, + StorageSlotType::Map => { + // For map slots, reconstruct from map entries + let entries = map_entries_by_slot.remove(slot_header.name()).unwrap_or_default(); + let storage_map = StorageMap::with_entries(entries)?; + StorageSlot::with_map(slot_header.name().clone(), storage_map) + }, + }; + slots.push(slot); + } + + Ok(AccountStorage::new(slots)?) +} diff --git a/crates/store/src/db/models/queries/accounts/tests.rs b/crates/store/src/db/models/queries/accounts/tests.rs new file mode 100644 index 0000000000..b68df7367e --- /dev/null +++ b/crates/store/src/db/models/queries/accounts/tests.rs @@ -0,0 +1,478 @@ +use assert_matches::assert_matches; +use diesel::{Connection, RunQueryDsl}; +use diesel_migrations::MigrationHarness; +use miden_lib::account::auth::AuthRpoFalcon512; +use miden_lib::transaction::TransactionKernel; +use miden_node_utils::fee::test_fee_params; +use miden_objects::account::auth::PublicKeyCommitment; +use miden_objects::account::{ + AccountBuilder, + AccountComponent, + AccountIdVersion, + AccountStorageMode, + AccountType, + StorageSlot, +}; +use miden_objects::{EMPTY_WORD, Word}; + +use super::*; +use crate::db::migrations::MIGRATIONS; + +fn setup_test_db() -> SqliteConnection { + let mut conn = + SqliteConnection::establish(":memory:").expect("Failed to create in-memory database"); + + conn.run_pending_migrations(MIGRATIONS).expect("Failed to run migrations"); + + conn +} + +fn create_test_account_with_storage() -> (Account, AccountId) { + // Create a simple public account with one value storage slot + let account_id = AccountId::dummy( + [1u8; 15], + AccountIdVersion::Version0, + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + ); + + let storage_value = Word::from([Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]); + let component_storage = vec![StorageSlot::Value(storage_value)]; + + let component = AccountComponent::compile( + "export.foo push.1 end", + TransactionKernel::assembler(), + component_storage, + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); + + let account = AccountBuilder::new([1u8; 32]) + .account_type(AccountType::RegularAccountImmutableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + (account, account_id) +} + +fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { + use miden_objects::block::BlockHeader; + + use crate::db::schema::block_headers; + + let block_header = BlockHeader::new( + 1_u8.into(), + Word::default(), + block_num, + Word::default(), + Word::default(), + Word::default(), + Word::default(), + Word::default(), + Word::default(), + Word::default(), + test_fee_params(), + 0_u8.into(), + ); + + diesel::insert_into(block_headers::table) + .values(( + block_headers::block_num.eq(i64::from(block_num.as_u32())), + block_headers::block_header.eq(block_header.to_bytes()), + )) + .execute(conn) + .expect("Failed to insert block header"); +} + +#[test] +fn test_upsert_accounts_inserts_storage_header() { + let mut conn = setup_test_db(); + let (account, account_id) = create_test_account_with_storage(); + + // Block 1 + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + let storage_commitment_original = account.storage().commitment(); + let storage_slots_len = account.storage().slots().len(); + let account_commitment = account.commitment(); + + // Create full state delta from the account + let delta = AccountDelta::try_from(account).unwrap(); + assert!(delta.is_full_state(), "Delta should be full state"); + + let account_update = + BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Delta(delta)); + + // Upsert account + let result = upsert_accounts(&mut conn, &[account_update], block_num); + assert!(result.is_ok(), "upsert_accounts failed: {:?}", result.err()); + assert_eq!(result.unwrap(), 1, "Expected 1 account to be inserted"); + + // Query storage header back + let queried_storage = select_latest_account_storage(&mut conn, account_id) + .expect("Failed to query storage header"); + + // Verify storage commitment matches + assert_eq!( + queried_storage.commitment(), + storage_commitment_original, + "Storage commitment mismatch" + ); + + // Verify number of slots matches + assert_eq!(queried_storage.slots().len(), storage_slots_len, "Storage slots count mismatch"); + + // Verify exactly 1 latest account with storage exists + let header_count: i64 = schema::accounts::table + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .filter(schema::accounts::storage_header.is_not_null()) + .count() + .get_result(&mut conn) + .expect("Failed to count accounts with storage"); + + assert_eq!(header_count, 1, "Expected exactly 1 latest account with storage"); +} + +#[test] +fn test_upsert_accounts_updates_is_latest_flag() { + let mut conn = setup_test_db(); + let (account, account_id) = create_test_account_with_storage(); + + // Block 1 and 2 + let block_num_1 = BlockNumber::from_epoch(0); + let block_num_2 = BlockNumber::from_epoch(1); + + insert_block_header(&mut conn, block_num_1); + insert_block_header(&mut conn, block_num_2); + + // Save storage commitment before moving account + let storage_commitment_1 = account.storage().commitment(); + let account_commitment_1 = account.commitment(); + + // First update with original account - full state delta + let delta_1 = AccountDelta::try_from(account).unwrap(); + + let account_update_1 = BlockAccountUpdate::new( + account_id, + account_commitment_1, + AccountUpdateDetails::Delta(delta_1), + ); + + upsert_accounts(&mut conn, &[account_update_1], block_num_1).expect("First upsert failed"); + + // Create modified account with different storage value + let storage_value_modified = + Word::from([Felt::new(10), Felt::new(20), Felt::new(30), Felt::new(40)]); + let component_storage_modified = vec![StorageSlot::Value(storage_value_modified)]; + + let component_2 = AccountComponent::compile( + "export.foo push.1 end", + TransactionKernel::assembler(), + component_storage_modified, + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); + + let account_2 = AccountBuilder::new([1u8; 32]) + .account_type(AccountType::RegularAccountImmutableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component_2) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + let storage_commitment_2 = account_2.storage().commitment(); + let account_commitment_2 = account_2.commitment(); + + // Second update with modified account - full state delta + let delta_2 = AccountDelta::try_from(account_2).unwrap(); + + let account_update_2 = BlockAccountUpdate::new( + account_id, + account_commitment_2, + AccountUpdateDetails::Delta(delta_2), + ); + + upsert_accounts(&mut conn, &[account_update_2], block_num_2).expect("Second upsert failed"); + + // Verify 2 total account rows exist (both historical records) + let total_accounts: i64 = schema::accounts::table + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .count() + .get_result(&mut conn) + .expect("Failed to count total accounts"); + + assert_eq!(total_accounts, 2, "Expected 2 total account records"); + + // Verify only 1 is marked as latest + let latest_accounts: i64 = schema::accounts::table + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)) + .count() + .get_result(&mut conn) + .expect("Failed to count latest accounts"); + + assert_eq!(latest_accounts, 1, "Expected exactly 1 latest account"); + + // Verify latest storage matches second update + let latest_storage = select_latest_account_storage(&mut conn, account_id) + .expect("Failed to query latest storage"); + + assert_eq!( + latest_storage.commitment(), + storage_commitment_2, + "Latest storage should match second update" + ); + + // Verify historical query returns first update + let storage_at_block_1 = select_account_storage_at_block(&mut conn, account_id, block_num_1) + .expect("Failed to query storage at block 1"); + + assert_eq!( + storage_at_block_1.commitment(), + storage_commitment_1, + "Storage at block 1 should match first update" + ); +} + +#[test] +fn test_upsert_accounts_with_incremental_delta() { + use std::collections::BTreeMap; + + use miden_objects::account::delta::{AccountStorageDelta, AccountVaultDelta}; + + let mut conn = setup_test_db(); + let (account, account_id) = create_test_account_with_storage(); + + let block_num_1 = BlockNumber::from_epoch(0); + let block_num_2 = BlockNumber::from_epoch(1); + + insert_block_header(&mut conn, block_num_1); + insert_block_header(&mut conn, block_num_2); + + // First update with full state + let storage_commitment_1 = account.storage().commitment(); + let account_commitment_1 = account.commitment(); + let nonce_1 = account.nonce(); + let delta_1 = AccountDelta::try_from(account).unwrap(); + + let account_update_1 = BlockAccountUpdate::new( + account_id, + account_commitment_1, + AccountUpdateDetails::Delta(delta_1), + ); + + upsert_accounts(&mut conn, &[account_update_1], block_num_1).expect("First upsert failed"); + + // Create incremental delta (only modify storage value slot 1) + let new_storage_value = + Word::from([Felt::new(100), Felt::new(200), Felt::new(300), Felt::new(400)]); + + let mut storage_delta_values = BTreeMap::new(); + storage_delta_values.insert(1u8, new_storage_value); // Update slot 1 (component storage) + + let storage_delta = AccountStorageDelta::from_parts(storage_delta_values, BTreeMap::new()) + .expect("Failed to create storage delta"); + let incremental_delta = + AccountDelta::new(account_id, storage_delta, AccountVaultDelta::default(), nonce_1) + .expect("Failed to create incremental delta"); + + // Reconstruct expected account after delta + let account_after = reconstruct_full_account_from_db(&mut conn, account_id) + .expect("Failed to reconstruct account"); + let mut expected_account = account_after.clone(); + expected_account + .apply_delta(&incremental_delta) + .expect("Failed to apply delta to expected account"); + + let storage_commitment_2 = expected_account.storage().commitment(); + let account_commitment_2 = expected_account.commitment(); + + let account_update_2 = BlockAccountUpdate::new( + account_id, + account_commitment_2, + AccountUpdateDetails::Delta(incremental_delta), + ); + + upsert_accounts(&mut conn, &[account_update_2], block_num_2) + .expect("Second upsert with incremental delta failed"); + + // Verify latest storage matches expected state + let latest_storage = select_latest_account_storage(&mut conn, account_id) + .expect("Failed to query latest storage"); + + assert_eq!( + latest_storage.commitment(), + storage_commitment_2, + "Storage commitment should match after incremental delta" + ); + + // Verify historical storage is preserved + let storage_at_block_1 = select_account_storage_at_block(&mut conn, account_id, block_num_1) + .expect("Failed to query storage at block 1"); + + assert_eq!( + storage_at_block_1.commitment(), + storage_commitment_1, + "Historical storage should be unchanged" + ); +} + +#[test] +fn test_upsert_accounts_with_multiple_storage_slots() { + let mut conn = setup_test_db(); + + // Create account with 3 storage slots + let account_id = AccountId::dummy( + [2u8; 15], + AccountIdVersion::Version0, + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + ); + + let slot_value_1 = Word::from([Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]); + let slot_value_2 = Word::from([Felt::new(5), Felt::new(6), Felt::new(7), Felt::new(8)]); + let slot_value_3 = Word::from([Felt::new(9), Felt::new(10), Felt::new(11), Felt::new(12)]); + + let component_storage = vec![ + StorageSlot::Value(slot_value_1), + StorageSlot::Value(slot_value_2), + StorageSlot::Value(slot_value_3), + ]; + + let component = AccountComponent::compile( + "export.foo push.1 end", + TransactionKernel::assembler(), + component_storage, + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); + + let account = AccountBuilder::new([2u8; 32]) + .account_type(AccountType::RegularAccountImmutableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + let storage_commitment = account.storage().commitment(); + let account_commitment = account.commitment(); + let delta = AccountDelta::try_from(account).unwrap(); + + let account_update = + BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Delta(delta)); + + upsert_accounts(&mut conn, &[account_update], block_num) + .expect("Upsert with multiple storage slots failed"); + + // Query back and verify + let queried_storage = + select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + + assert_eq!(queried_storage.commitment(), storage_commitment, "Storage commitment mismatch"); + + // Note: Auth component adds 1 storage slot, so 3 component slots + 1 auth = 4 total + assert_eq!( + queried_storage.slots().len(), + 4, + "Expected 4 storage slots (3 component + 1 auth)" + ); + + // Verify individual slot values (skipping auth slot at index 0) + assert_matches!( + queried_storage.slots().get(1).expect("Slot 1 should exist"), + &StorageSlot::Value(v) if v == slot_value_1, + "Slot 1 value mismatch" + ); + assert_matches!( + queried_storage.slots().get(2).expect("Slot 2 should exist"), + &StorageSlot::Value(v) if v == slot_value_2, + "Slot 2 value mismatch" + ); + assert_matches!( + queried_storage.slots().get(3).expect("Slot 3 should exist"), + &StorageSlot::Value(v) if v == slot_value_3, + "Slot 3 value mismatch" + ); +} + +#[test] +fn test_upsert_accounts_with_empty_storage() { + let mut conn = setup_test_db(); + + // Create account with no storage slots + let account_id = AccountId::dummy( + [3u8; 15], + AccountIdVersion::Version0, + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + ); + + let component = AccountComponent::compile( + "export.foo push.1 end", + TransactionKernel::assembler(), + vec![], // Empty storage + ) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); + + let account = AccountBuilder::new([3u8; 32]) + .account_type(AccountType::RegularAccountImmutableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + let storage_commitment = account.storage().commitment(); + let account_commitment = account.commitment(); + let delta = AccountDelta::try_from(account).unwrap(); + + let account_update = + BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Delta(delta)); + + upsert_accounts(&mut conn, &[account_update], block_num) + .expect("Upsert with empty storage failed"); + + // Query back and verify + let queried_storage = + select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + + assert_eq!( + queried_storage.commitment(), + storage_commitment, + "Storage commitment mismatch for empty storage" + ); + + // Note: Auth component adds 1 storage slot, so even "empty" accounts have 1 slot + assert_eq!(queried_storage.slots().len(), 1, "Expected 1 storage slot (auth component)"); + + // Verify the storage header blob exists in database + let storage_header_exists: Option = SelectDsl::select( + schema::accounts::table + .filter(schema::accounts::account_id.eq(account_id.to_bytes())) + .filter(schema::accounts::is_latest.eq(true)), + schema::accounts::storage_header.is_not_null(), + ) + .first(&mut conn) + .optional() + .expect("Failed to check storage header existence"); + + assert_eq!( + storage_header_exists, + Some(true), + "Storage header blob should exist even for empty storage" + ); +} diff --git a/crates/store/src/db/schema.rs b/crates/store/src/db/schema.rs index 6f36594b9a..90c48380d1 100644 --- a/crates/store/src/db/schema.rs +++ b/crates/store/src/db/schema.rs @@ -27,9 +27,9 @@ diesel::table! { network_account_id_prefix -> Nullable, account_commitment -> Binary, code_commitment -> Nullable, - storage -> Nullable, - vault -> Nullable, nonce -> Nullable, + storage_header -> Nullable, + vault_root -> Nullable, block_num -> BigInt, is_latest -> Bool, } diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 413f8a5242..488b9232de 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -12,6 +12,7 @@ use miden_protocol::account::delta::AccountUpdateDetails; use miden_protocol::account::{ Account, AccountBuilder, + AccountCode, AccountComponent, AccountDelta, AccountId, @@ -21,6 +22,7 @@ use miden_protocol::account::{ AccountType, AccountVaultDelta, StorageSlot, + StorageSlotContent, StorageSlotDelta, StorageSlotName, }; @@ -61,6 +63,7 @@ use miden_protocol::transaction::{ TransactionHeader, TransactionId, }; +use miden_protocol::utils::{Deserializable, Serializable}; use miden_protocol::{EMPTY_WORD, Felt, FieldElement, Word, ZERO}; use miden_standards::account::auth::AuthRpoFalcon512; use miden_standards::code_builder::CodeBuilder; @@ -464,27 +467,25 @@ fn sql_unconsumed_network_notes() { create_block(&mut conn, 1.into()); // Create an unconsumed note in each block. - let notes = (0..2) - .map(|i: u32| { - let note = NoteRecord { - block_num: 0.into(), // Created on same block. - note_index: BlockNoteIndex::new(0, i as usize).unwrap(), - note_id: num_to_word(i.into()), - note_commitment: num_to_word(i.into()), - metadata: NoteMetadata::new( - account_note.0, - NoteType::Public, - NoteTag::from_account_id(account_note.0), - NoteExecutionHint::none(), - Felt::default(), - ) - .unwrap(), - details: None, - inclusion_path: SparseMerklePath::default(), - }; - (note, Some(num_to_nullifier(i.into()))) - }) - .collect::>(); + let notes = Vec::from_iter((0..2).map(|i: u32| { + let note = NoteRecord { + block_num: 0.into(), // Created on same block. + note_index: BlockNoteIndex::new(0, i as usize).unwrap(), + note_id: num_to_word(i.into()), + note_commitment: num_to_word(i.into()), + metadata: NoteMetadata::new( + account_note.0, + NoteType::Public, + NoteTag::from_account_id(account_note.0), + NoteExecutionHint::none(), + Felt::default(), + ) + .unwrap(), + details: None, + inclusion_path: SparseMerklePath::default(), + }; + (note, Some(num_to_nullifier(i.into()))) + })); queries::insert_scripts(&mut conn, notes.iter().map(|(note, _)| note)).unwrap(); queries::insert_notes(&mut conn, ¬es).unwrap(); @@ -1165,8 +1166,7 @@ fn sql_account_storage_map_values_insertion() { let mut map1 = StorageMapDelta::default(); map1.insert(key1, value1); map1.insert(key2, value2); - let delta1: BTreeMap<_, _> = - [(slot_name.clone(), StorageSlotDelta::Map(map1))].into_iter().collect(); + let delta1 = BTreeMap::from_iter([(slot_name.clone(), StorageSlotDelta::Map(map1))]); let storage1 = AccountStorageDelta::from_raw(delta1); let delta1 = AccountDelta::new(account_id, storage1, AccountVaultDelta::default(), Felt::ONE).unwrap(); @@ -1326,6 +1326,30 @@ fn mock_block_account_update(account_id: AccountId, num: u64) -> BlockAccountUpd BlockAccountUpdate::new(account_id, num_to_word(num), AccountUpdateDetails::Private) } +// Helper function to create account with specific code for tests +fn create_account_with_code(code_str: &str, seed: [u8; 32]) -> Account { + let component_storage = vec![ + StorageSlot::with_value(StorageSlotName::mock(0), Word::empty()), + StorageSlot::with_value(StorageSlotName::mock(1), num_to_word(1)), + ]; + + let account_component_code = CodeBuilder::default() + .compile_component_code("test::interface", code_str) + .unwrap(); + + let component = AccountComponent::new(account_component_code, component_storage) + .unwrap() + .with_supported_type(AccountType::RegularAccountUpdatableCode); + + AccountBuilder::new(seed) + .account_type(AccountType::RegularAccountUpdatableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap() +} + fn mock_block_transaction(account_id: AccountId, num: u64) -> TransactionHeader { let initial_state_commitment = Word::try_from([num, 0, 0, 0]).unwrap(); let final_account_commitment = Word::try_from([0, num, 0, 0]).unwrap(); @@ -1428,6 +1452,166 @@ fn mock_account_code_and_storage( .unwrap() } +// STORAGE RECONSTRUCTION TESTS +// ================================================================================================ + +#[test] +fn test_select_account_code_at_block() { + let mut conn = create_db(); + + let block_num_1 = BlockNumber::from(1); + + // Create block 1 + create_block(&mut conn, block_num_1); + + // Create an account with code at block 1 using the existing mock function + let account = mock_account_code_and_storage( + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + [], + None, + ); + + // Use the actual account ID from the created account + let account_id = account.id(); + + // Get the code bytes before inserting + let expected_code = account.code().to_bytes(); + + // Insert the account at block 1 + queries::upsert_accounts( + &mut conn, + &[BlockAccountUpdate::new( + account_id, + account.commitment(), + AccountUpdateDetails::Delta(AccountDelta::try_from(account).unwrap()), + )], + block_num_1, + ) + .unwrap(); + + // Query code at block 1 - should return the code + let code_at_1 = queries::select_account_code_at_block(&mut conn, account_id, block_num_1) + .unwrap() + .expect("Code should exist at block 1"); + assert_eq!(code_at_1, expected_code); + + // Query code for non-existent account - should return None + let other_account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + let code_other = + queries::select_account_code_at_block(&mut conn, other_account_id, block_num_1).unwrap(); + assert!(code_other.is_none(), "Code should not exist for non-existent account"); +} + +#[test] +fn test_select_account_code_at_block_with_updates() { + let mut conn = create_db(); + + let block_num_1 = BlockNumber::from(1); + let block_num_2 = BlockNumber::from(2); + let block_num_3 = BlockNumber::from(3); + + // Create all blocks + create_block(&mut conn, block_num_1); + create_block(&mut conn, block_num_2); + create_block(&mut conn, block_num_3); + + // Create initial account with code v1 at block 1 + let code_v1_str = "\ + pub proc account_procedure_1 + push.1.2 + add + end + "; + let account_v1 = create_account_with_code(code_v1_str, [1u8; 32]); + let account_id = account_v1.id(); + let code_v1 = account_v1.code().to_bytes(); + + // Insert the account at block 1 + queries::upsert_accounts( + &mut conn, + &[BlockAccountUpdate::new( + account_id, + account_v1.commitment(), + AccountUpdateDetails::Delta(AccountDelta::try_from(account_v1).unwrap()), + )], + block_num_1, + ) + .unwrap(); + + // Create account with different code v2 at block 2 + let code_v2_str = "\ + pub proc account_procedure_1 + push.3.4 + mul + end + "; + let account_v2 = create_account_with_code(code_v2_str, [1u8; 32]); // Same seed to keep same account_id + let code_v2 = account_v2.code().to_bytes(); + + // Verify that the codes are actually different + assert_ne!( + code_v1, code_v2, + "Test setup error: codes should be different for different code strings" + ); + + // Insert the updated account at block 2 + queries::upsert_accounts( + &mut conn, + &[BlockAccountUpdate::new( + account_id, + account_v2.commitment(), + AccountUpdateDetails::Delta(AccountDelta::try_from(account_v2).unwrap()), + )], + block_num_2, + ) + .unwrap(); + + // Create account with different code v3 at block 3 + let code_v3_str = "\ + pub proc account_procedure_1 + push.5.6 + sub + end + "; + let account_v3 = create_account_with_code(code_v3_str, [1u8; 32]); // Same seed to keep same account_id + let code_v3 = account_v3.code().to_bytes(); + + // Verify that v3 code is different from v2 and v1 + assert_ne!(code_v2, code_v3, "Test setup error: v3 code should differ from v2"); + assert_ne!(code_v1, code_v3, "Test setup error: v3 code should differ from v1"); + + // Insert the updated account at block 3 + queries::upsert_accounts( + &mut conn, + &[BlockAccountUpdate::new( + account_id, + account_v3.commitment(), + AccountUpdateDetails::Delta(AccountDelta::try_from(account_v3).unwrap()), + )], + block_num_3, + ) + .unwrap(); + + // Test: Query code at block 1 - should return v1 code + let code_at_1 = queries::select_account_code_at_block(&mut conn, account_id, block_num_1) + .unwrap() + .expect("Code should exist at block 1"); + assert_eq!(code_at_1, code_v1, "Block 1 should return v1 code"); + + // Test: Query code at block 2 - should return v2 code (even though we're at block 3) + let code_at_2 = queries::select_account_code_at_block(&mut conn, account_id, block_num_2) + .unwrap() + .expect("Code should exist at block 2"); + assert_eq!(code_at_2, code_v2, "Block 2 should return v2 code"); + + // Test: Query code at block 3 - should return v3 code + let code_at_3 = queries::select_account_code_at_block(&mut conn, account_id, block_num_3) + .unwrap() + .expect("Code should exist at block 3"); + assert_eq!(code_at_3, code_v3, "Block 3 should return v3 code"); +} + // GENESIS REGRESSION TESTS // ================================================================================================ @@ -1682,3 +1866,504 @@ fn regression_1461_full_state_delta_inserts_vault_assets() { assert_eq!(vault_asset.asset, Some(expected_asset)); assert_eq!(vault_asset.vault_key, expected_asset.vault_key()); } + +// SERIALIZATION SYMMETRY TESTS +// ================================================================================================ +// +// These tests ensure that `to_bytes` and `from_bytes`/`read_from_bytes` are symmetric for all +// types used in database operations. This guarantees that data inserted into the database can +// always be correctly retrieved. + +#[test] +fn serialization_symmetry_core_types() { + // AccountId + let account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + let bytes = account_id.to_bytes(); + let restored = AccountId::read_from_bytes(&bytes).unwrap(); + assert_eq!(account_id, restored, "AccountId serialization must be symmetric"); + + // Word + let word = num_to_word(0x1234_5678_9ABC_DEF0); + let bytes = word.to_bytes(); + let restored = Word::read_from_bytes(&bytes).unwrap(); + assert_eq!(word, restored, "Word serialization must be symmetric"); + + // Nullifier + let nullifier = num_to_nullifier(0xDEAD_BEEF); + let bytes = nullifier.to_bytes(); + let restored = Nullifier::read_from_bytes(&bytes).unwrap(); + assert_eq!(nullifier, restored, "Nullifier serialization must be symmetric"); + + // TransactionId + let tx_id = TransactionId::new(num_to_word(1), num_to_word(2), num_to_word(3), num_to_word(4)); + let bytes = tx_id.to_bytes(); + let restored = TransactionId::read_from_bytes(&bytes).unwrap(); + assert_eq!(tx_id, restored, "TransactionId serialization must be symmetric"); + + // NoteId + let note_id = NoteId::new(num_to_word(1), num_to_word(2)); + let bytes = note_id.to_bytes(); + let restored = NoteId::read_from_bytes(&bytes).unwrap(); + assert_eq!(note_id, restored, "NoteId serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_block_header() { + let block_header = BlockHeader::new( + 1_u8.into(), + num_to_word(2), + 3.into(), + num_to_word(4), + num_to_word(5), + num_to_word(6), + num_to_word(7), + num_to_word(8), + num_to_word(9), + SecretKey::new().public_key(), + test_fee_params(), + 11_u8.into(), + ); + + let bytes = block_header.to_bytes(); + let restored = BlockHeader::read_from_bytes(&bytes).unwrap(); + assert_eq!(block_header, restored, "BlockHeader serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_assets() { + let faucet_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); + + // FungibleAsset + let fungible = FungibleAsset::new(faucet_id, 1000).unwrap(); + let asset: Asset = fungible.into(); + let bytes = asset.to_bytes(); + let restored = Asset::read_from_bytes(&bytes).unwrap(); + assert_eq!(asset, restored, "Asset (fungible) serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_account_code() { + let account = mock_account_code_and_storage( + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + [], + None, + ); + + let code = account.code(); + let bytes = code.to_bytes(); + let restored = AccountCode::read_from_bytes(&bytes).unwrap(); + assert_eq!(*code, restored, "AccountCode serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_sparse_merkle_path() { + let path = SparseMerklePath::default(); + let bytes = path.to_bytes(); + let restored = SparseMerklePath::read_from_bytes(&bytes).unwrap(); + assert_eq!(path, restored, "SparseMerklePath serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_note_metadata() { + let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + // Use a tag that roundtrips properly - NoteTag::LocalAny stores the full u32 including type + // bits + let tag = NoteTag::from_account_id(sender); + let metadata = NoteMetadata::new( + sender, + NoteType::Public, + tag, + NoteExecutionHint::always(), + Felt::new(42), + ) + .unwrap(); + + let bytes = metadata.to_bytes(); + let restored = NoteMetadata::read_from_bytes(&bytes).unwrap(); + assert_eq!(metadata, restored, "NoteMetadata serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_nullifier_vec() { + let nullifiers: Vec = (0..5).map(num_to_nullifier).collect(); + let bytes = nullifiers.to_bytes(); + let restored: Vec = Deserializable::read_from_bytes(&bytes).unwrap(); + assert_eq!(nullifiers, restored, "Vec serialization must be symmetric"); +} + +#[test] +fn serialization_symmetry_note_id_vec() { + let note_ids: Vec = + (0..5).map(|i| NoteId::new(num_to_word(i), num_to_word(i + 100))).collect(); + let bytes = note_ids.to_bytes(); + let restored: Vec = Deserializable::read_from_bytes(&bytes).unwrap(); + assert_eq!(note_ids, restored, "Vec serialization must be symmetric"); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_block_header() { + let mut conn = create_db(); + + let block_header = BlockHeader::new( + 1_u8.into(), + num_to_word(2), + BlockNumber::from(42), + num_to_word(4), + num_to_word(5), + num_to_word(6), + num_to_word(7), + num_to_word(8), + num_to_word(9), + SecretKey::new().public_key(), + test_fee_params(), + 11_u8.into(), + ); + + // Insert + queries::insert_block_header(&mut conn, &block_header).unwrap(); + + // Retrieve + let retrieved = + queries::select_block_header_by_block_num(&mut conn, Some(block_header.block_num())) + .unwrap() + .expect("Block header should exist"); + + assert_eq!(block_header, retrieved, "BlockHeader DB roundtrip must be symmetric"); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_nullifiers() { + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + let nullifiers: Vec = (0..5).map(|i| num_to_nullifier(i << 48)).collect(); + + // Insert + queries::insert_nullifiers_for_block(&mut conn, &nullifiers, block_num).unwrap(); + + // Retrieve + let retrieved = queries::select_all_nullifiers(&mut conn).unwrap(); + + assert_eq!(nullifiers.len(), retrieved.len(), "Should retrieve same number of nullifiers"); + for (orig, info) in nullifiers.iter().zip(retrieved.iter()) { + assert_eq!(*orig, info.nullifier, "Nullifier DB roundtrip must be symmetric"); + assert_eq!(block_num, info.block_num, "Block number must match"); + } +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_account() { + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + let account = mock_account_code_and_storage( + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + [], + Some([99u8; 32]), + ); + let account_id = account.id(); + let account_commitment = account.commitment(); + + // Insert with full delta (like genesis) + let account_delta = AccountDelta::try_from(account.clone()).unwrap(); + let block_update = BlockAccountUpdate::new( + account_id, + account_commitment, + AccountUpdateDetails::Delta(account_delta), + ); + queries::upsert_accounts(&mut conn, &[block_update], block_num).unwrap(); + + // Retrieve + let retrieved = queries::select_all_accounts(&mut conn).unwrap(); + assert_eq!(retrieved.len(), 1, "Should have one account"); + + let retrieved_info = &retrieved[0]; + assert_eq!( + retrieved_info.summary.account_id, account_id, + "AccountId DB roundtrip must be symmetric" + ); + assert_eq!( + retrieved_info.summary.account_commitment, account_commitment, + "Account commitment DB roundtrip must be symmetric" + ); + assert_eq!(retrieved_info.summary.block_num, block_num, "Block number must match"); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_notes() { + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + queries::upsert_accounts(&mut conn, &[mock_block_account_update(sender, 0)], block_num) + .unwrap(); + + let new_note = create_note(sender); + let note_index = BlockNoteIndex::new(0, 0).unwrap(); + + let note = NoteRecord { + block_num, + note_index, + note_id: new_note.id().as_word(), + note_commitment: new_note.commitment(), + metadata: *new_note.metadata(), + details: Some(NoteDetails::from(&new_note)), + inclusion_path: SparseMerklePath::default(), + }; + + // Insert + queries::insert_scripts(&mut conn, [¬e]).unwrap(); + queries::insert_notes(&mut conn, &[(note.clone(), None)]).unwrap(); + + // Retrieve + let note_ids = vec![NoteId::from_raw(note.note_id)]; + let retrieved = queries::select_notes_by_id(&mut conn, ¬e_ids).unwrap(); + + assert_eq!(retrieved.len(), 1, "Should have one note"); + let retrieved_note = &retrieved[0]; + + assert_eq!(note.note_id, retrieved_note.note_id, "NoteId DB roundtrip must be symmetric"); + assert_eq!( + note.note_commitment, retrieved_note.note_commitment, + "Note commitment DB roundtrip must be symmetric" + ); + assert_eq!( + note.metadata, retrieved_note.metadata, + "Metadata DB roundtrip must be symmetric" + ); + assert_eq!( + note.inclusion_path, retrieved_note.inclusion_path, + "Inclusion path DB roundtrip must be symmetric" + ); + assert_eq!( + note.details, retrieved_note.details, + "Note details DB roundtrip must be symmetric" + ); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_transactions() { + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + let account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + queries::upsert_accounts(&mut conn, &[mock_block_account_update(account_id, 1)], block_num) + .unwrap(); + + let tx = mock_block_transaction(account_id, 1); + let ordered_tx = OrderedTransactionHeaders::new_unchecked(vec![tx.clone()]); + + // Insert + queries::insert_transactions(&mut conn, block_num, &ordered_tx).unwrap(); + + // Retrieve + let retrieved = queries::select_transactions_by_accounts_and_block_range( + &mut conn, + &[account_id], + BlockNumber::from(0)..=BlockNumber::from(2), + ) + .unwrap(); + + assert_eq!(retrieved.len(), 1, "Should have one transaction"); + let retrieved_tx = &retrieved[0]; + + assert_eq!( + tx.account_id(), + retrieved_tx.account_id, + "AccountId DB roundtrip must be symmetric" + ); + assert_eq!( + tx.id(), + retrieved_tx.transaction_id, + "TransactionId DB roundtrip must be symmetric" + ); + assert_eq!(block_num, retrieved_tx.block_num, "Block number must match"); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_vault_assets() { + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + let faucet_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + + // Create account first + queries::upsert_accounts(&mut conn, &[mock_block_account_update(account_id, 0)], block_num) + .unwrap(); + + let fungible_asset = FungibleAsset::new(faucet_id, 5000).unwrap(); + let asset: Asset = fungible_asset.into(); + let vault_key = asset.vault_key(); + + // Insert vault asset + queries::insert_account_vault_asset(&mut conn, account_id, block_num, vault_key, Some(asset)) + .unwrap(); + + // Retrieve + let (_, vault_assets) = queries::select_account_vault_assets( + &mut conn, + account_id, + BlockNumber::GENESIS..=block_num, + ) + .unwrap(); + + assert_eq!(vault_assets.len(), 1, "Should have one vault asset"); + let retrieved = &vault_assets[0]; + + assert_eq!(retrieved.asset, Some(asset), "Asset DB roundtrip must be symmetric"); + assert_eq!(retrieved.vault_key, vault_key, "VaultKey DB roundtrip must be symmetric"); + assert_eq!(retrieved.block_num, block_num, "Block number must match"); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_storage_map_values() { + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); + let slot_name = StorageSlotName::mock(5); + let key = num_to_word(12345); + let value = num_to_word(67890); + + // Insert + queries::insert_account_storage_map_value( + &mut conn, + account_id, + block_num, + slot_name.clone(), + key, + value, + ) + .unwrap(); + + // Retrieve + let page = queries::select_account_storage_map_values( + &mut conn, + account_id, + BlockNumber::GENESIS..=block_num, + ) + .unwrap(); + + assert_eq!(page.values.len(), 1, "Should have one storage map value"); + let retrieved = &page.values[0]; + + assert_eq!(retrieved.slot_name, slot_name, "StorageSlotName DB roundtrip must be symmetric"); + assert_eq!(retrieved.key, key, "Key (Word) DB roundtrip must be symmetric"); + assert_eq!(retrieved.value, value, "Value (Word) DB roundtrip must be symmetric"); + assert_eq!(retrieved.block_num, block_num, "Block number must match"); +} + +#[test] +#[miden_node_test_macro::enable_logging] +fn db_roundtrip_account_storage_with_maps() { + use miden_protocol::account::StorageMap; + + let mut conn = create_db(); + let block_num = BlockNumber::from(1); + create_block(&mut conn, block_num); + + // Create storage with both value slots and map slots + let storage_map = StorageMap::with_entries(vec![ + ( + Word::from([Felt::new(1), Felt::ZERO, Felt::ZERO, Felt::ZERO]), + Word::from([Felt::new(10), Felt::new(20), Felt::new(30), Felt::new(40)]), + ), + ( + Word::from([Felt::new(2), Felt::ZERO, Felt::ZERO, Felt::ZERO]), + Word::from([Felt::new(50), Felt::new(60), Felt::new(70), Felt::new(80)]), + ), + ]) + .unwrap(); + + let component_storage = vec![ + StorageSlot::with_value(StorageSlotName::mock(0), num_to_word(42)), + StorageSlot::with_map(StorageSlotName::mock(1), storage_map), + StorageSlot::with_empty_value(StorageSlotName::mock(2)), + ]; + + let component_code = "pub proc foo push.1 end"; + let account_component_code = CodeBuilder::default() + .compile_component_code("test::interface", component_code) + .unwrap(); + let account_component = AccountComponent::new(account_component_code, component_storage) + .unwrap() + .with_supports_all_types(); + + let account = AccountBuilder::new([50u8; 32]) + .account_type(AccountType::RegularAccountUpdatableCode) + .storage_mode(AccountStorageMode::Public) + .with_component(account_component) + .with_auth_component(AuthRpoFalcon512::new(PublicKeyCommitment::from(EMPTY_WORD))) + .build_existing() + .unwrap(); + + let account_id = account.id(); + let original_storage = account.storage().clone(); + let original_commitment = original_storage.to_commitment(); + + // Insert the account (this should store header + map values separately) + let account_delta = AccountDelta::try_from(account.clone()).unwrap(); + let block_update = BlockAccountUpdate::new( + account_id, + account.commitment(), + AccountUpdateDetails::Delta(account_delta), + ); + queries::upsert_accounts(&mut conn, &[block_update], block_num).unwrap(); + + // Retrieve the storage using select_latest_account_storage (reconstructs from header + map + // values) + let retrieved_storage = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + let retrieved_commitment = retrieved_storage.to_commitment(); + + // Verify the commitment matches (this proves the reconstruction is correct) + assert_eq!( + original_commitment, retrieved_commitment, + "Storage commitment must match after DB roundtrip" + ); + + // Verify slot count matches + assert_eq!( + original_storage.slots().len(), + retrieved_storage.slots().len(), + "Number of slots must match" + ); + + // Verify each slot + for (original_slot, retrieved_slot) in + original_storage.slots().iter().zip(retrieved_storage.slots().iter()) + { + assert_eq!(original_slot.name(), retrieved_slot.name(), "Slot names must match"); + assert_eq!(original_slot.slot_type(), retrieved_slot.slot_type(), "Slot types must match"); + + match (original_slot.content(), retrieved_slot.content()) { + (StorageSlotContent::Value(orig), StorageSlotContent::Value(retr)) => { + assert_eq!(orig, retr, "Value slot contents must match"); + }, + (StorageSlotContent::Map(orig_map), StorageSlotContent::Map(retr_map)) => { + assert_eq!(orig_map.root(), retr_map.root(), "Map slot roots must match"); + for (key, value) in orig_map.entries() { + let retrieved_value = retr_map.get(key); + assert_eq!(*value, retrieved_value, "Map entry for key {:?} must match", key); + } + }, + // The slot_type assertion above guarantees matching variants, so this is unreachable + _ => unreachable!(), + } + } +} diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 4399918ba6..42a0fe32d9 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -8,9 +8,10 @@ use miden_node_proto::errors::{ConversionError, GrpcError}; use miden_node_utils::limiter::QueryLimitError; use miden_protocol::account::AccountId; use miden_protocol::block::BlockNumber; +use miden_protocol::crypto::merkle::MerkleError; use miden_protocol::crypto::merkle::mmr::MmrError; use miden_protocol::crypto::utils::DeserializationError; -use miden_protocol::note::Nullifier; +use miden_protocol::note::{NoteId, Nullifier}; use miden_protocol::transaction::OutputNote; use miden_protocol::{ AccountDeltaError, @@ -21,6 +22,7 @@ use miden_protocol::{ FeeError, NoteError, NullifierTreeError, + StorageMapError, Word, }; use thiserror::Error; @@ -56,11 +58,13 @@ pub enum DatabaseError { #[error("I/O error")] IoError(#[from] io::Error), #[error("merkle error")] - MerkleError(#[from] miden_protocol::crypto::merkle::MerkleError), + MerkleError(#[from] MerkleError), #[error("network account error")] NetworkAccountError(#[from] NetworkAccountError), #[error("note error")] NoteError(#[from] NoteError), + #[error("storage map error")] + StorageMapError(#[from] StorageMapError), #[error("setup deadpool connection pool failed")] Deadpool(#[from] deadpool::managed::PoolError), #[error("setup deadpool connection pool failed")] @@ -98,16 +102,18 @@ pub enum DatabaseError { AccountNotFoundInDb(AccountId), #[error("account {0} state at block height {1} not found")] AccountAtBlockHeightNotFoundInDb(AccountId, BlockNumber), + #[error("block {0} not found in database")] + BlockNotFound(BlockNumber), #[error("historical block {block_num} not available: {reason}")] HistoricalBlockNotAvailable { block_num: BlockNumber, reason: String }, #[error("accounts {0:?} not found")] AccountsNotFoundInDb(Vec), #[error("account {0} is not on the chain")] AccountNotPublic(AccountId), - #[error("account {0} details missing")] - AccountDetailsMissing(AccountId), #[error("invalid block parameters: block_from ({from}) > block_to ({to})")] InvalidBlockRange { from: BlockNumber, to: BlockNumber }, + #[error("invalid storage slot type: {0}")] + InvalidStorageSlotType(i32), #[error("data corrupted: {0}")] DataCorrupted(String), #[error("SQLite pool interaction failed: {0}")] @@ -175,6 +181,8 @@ impl From for Status { pub enum StateInitializationError { #[error("account tree IO error: {0}")] AccountTreeIoError(String), + #[error("nullifier tree IO error: {0}")] + NullifierTreeIoError(String), #[error("database error")] DatabaseError(#[from] DatabaseError), #[error("failed to create nullifier tree")] @@ -248,6 +256,8 @@ pub enum InvalidBlockError { NewBlockNullifierAlreadySpent(#[source] NullifierTreeError), #[error("duplicate account ID prefix in new block")] NewBlockDuplicateAccountIdPrefix(#[source] AccountTreeError), + #[error("failed to build note tree: {0}")] + FailedToBuildNoteTree(String), } #[derive(Error, Debug)] @@ -447,9 +457,9 @@ pub enum GetNotesByIdError { #[error("malformed note ID")] DeserializationFailed(#[from] ConversionError), #[error("note {0} not found")] - NoteNotFound(miden_protocol::note::NoteId), + NoteNotFound(NoteId), #[error("note {0} is not public")] - NoteNotPublic(miden_protocol::note::NoteId), + NoteNotPublic(NoteId), } // GET NOTE SCRIPT BY ROOT ERRORS diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index ab308569f1..45c4049ec4 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -23,11 +23,11 @@ use miden_node_proto::domain::account::{ use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::formatting::format_array; -use miden_protocol::account::{AccountHeader, AccountId, StorageSlot, StorageSlotContent}; +use miden_protocol::account::{AccountId, StorageSlotContent}; use miden_protocol::block::account_tree::{AccountTree, AccountWitness, account_id_to_smt_key}; use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain, ProvenBlock}; -use miden_protocol::crypto::merkle::mmr::{Forest, Mmr, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; +use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrPeaks, MmrProof, PartialMmr}; use miden_protocol::crypto::merkle::smt::{ LargeSmt, LargeSmtError, @@ -100,7 +100,16 @@ where } } -/// The rollup state +// CHAIN STATE +// ================================================================================================ + +/// The chain state. +/// +/// The chain state consists of three main components: +/// - A persistent database that stores notes, nullifiers, recent account states, and related data. +/// - In-memory data structures contain Merkle paths for various objects - e.g., all accounts, +/// nullifiers, public account vaults and storage, MMR of all block headers. +/// - Raw block data for all blocks that is stored on disk as flat files. pub struct State { /// The database which stores block headers, nullifiers, notes, and the latest states of /// accounts. @@ -120,6 +129,9 @@ pub struct State { } impl State { + // CONSTRUCTOR + // -------------------------------------------------------------------------------------------- + /// Loads the state from the `db`. #[instrument(target = COMPONENT, skip_all)] pub async fn load(data_path: &Path) -> Result { @@ -136,21 +148,12 @@ impl State { .await .map_err(StateInitializationError::DatabaseLoadError)?; - let chain_mmr = load_mmr(&mut db).await?; - let block_headers = db.select_all_block_headers().await?; - let latest_block_num = block_headers - .last() - .map_or(BlockNumber::GENESIS, miden_protocol::block::BlockHeader::block_num); + let blockchain = load_mmr(&mut db).await?; + let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); let account_tree = load_account_tree(&mut db, latest_block_num).await?; let nullifier_tree = load_nullifier_tree(&mut db).await?; - let inner = RwLock::new(InnerState { - nullifier_tree, - // SAFETY: We assume the loaded MMR is valid and does not have more than u32::MAX - // entries. - blockchain: Blockchain::from_mmr_unchecked(chain_mmr), - account_tree, - }); + let inner = RwLock::new(InnerState { nullifier_tree, blockchain, account_tree }); let writer = Mutex::new(()); let db = Arc::new(db); @@ -158,6 +161,9 @@ impl State { Ok(Self { db, block_store, inner, writer }) } + // STATE MUTATOR + // -------------------------------------------------------------------------------------------- + /// Apply changes of a new block to the DB and in-memory data structures. /// /// ## Note on state consistency @@ -200,7 +206,7 @@ impl State { } let block_num = header.block_num(); - let block_commitment = block.header().commitment(); + let block_commitment = header.commitment(); // ensures the right block header is being processed let prev_block = self @@ -249,7 +255,7 @@ impl State { .body() .created_nullifiers() .iter() - .filter(|&n| inner.nullifier_tree.get_block_num(n).is_some()) + .filter(|&nullifier| inner.nullifier_tree.get_block_num(nullifier).is_some()) .copied() .collect(); if !duplicate_nullifiers.is_empty() { @@ -418,6 +424,9 @@ impl State { Ok(()) } + // STATE ACCESSORS + // -------------------------------------------------------------------------------------------- + /// Queries a [BlockHeader] from the database, and returns it alongside its inclusion proof. /// /// If [None] is given as the value of `block_num`, the data for the latest [BlockHeader] is @@ -926,7 +935,7 @@ impl State { return Err(DatabaseError::AccountNotPublic(account_id)); } - let (block_num, witness) = self.get_block_witness(block_num, account_id).await?; + let (block_num, witness) = self.get_account_witness(block_num, account_id).await?; let details = if let Some(request) = details { Some(self.fetch_public_account_details(account_id, block_num, request).await?) @@ -941,7 +950,7 @@ impl State { /// /// If `block_num` is provided, returns the witness at that historical block, /// if not present, returns the witness at the latest block. - async fn get_block_witness( + async fn get_account_witness( &self, block_num: Option, account_id: AccountId, @@ -987,67 +996,68 @@ impl State { storage_requests, } = detail_request; - let account_info = self.db.select_historical_account_at(account_id, block_num).await?; + if !account_id.has_public_state() { + return Err(DatabaseError::AccountNotPublic(account_id)); + } + + // Validate block exists in the blockchain before querying the database + self.validate_block_exists(block_num).await?; - // If we get a query for a public account but the details are missing from the database, - // it indicates an inconsistent state in the database. - let Some(account) = account_info.details else { - return Err(DatabaseError::AccountDetailsMissing(account_id)); + let account_header = self + .db + .select_account_header_at_block(account_id, block_num) + .await? + .ok_or_else(|| DatabaseError::AccountNotPublic(account_id))?; + + let account_code = match code_commitment { + Some(commitment) if commitment == account_header.code_commitment() => None, + Some(_) => self.db.select_account_code_at_block(account_id, block_num).await?, + None => None, }; - let storage_header = account.storage().to_header(); + let vault_details = match asset_vault_commitment { + Some(commitment) if commitment == account_header.vault_root() => { + AccountVaultDetails::empty() + }, + Some(_) | None if asset_vault_commitment.is_some() => { + let vault_assets = + self.db.select_account_vault_at_block(account_id, block_num).await?; + AccountVaultDetails::from_assets(vault_assets) + }, + _ => AccountVaultDetails::empty(), + }; + // TODO: don't load the entire store at once, load what is required + let store = self.db.select_account_storage_at_block(account_id, block_num).await?; + let storage_header = store.to_header(); let mut storage_map_details = Vec::::with_capacity(storage_requests.len()); for StorageMapRequest { slot_name, slot_data } in storage_requests { - let Some(StorageSlotContent::Map(storage_map)) = - account.storage().get(&slot_name).map(StorageSlot::content) - else { - return Err(AccountError::StorageSlotNotMap(slot_name).into()); + let Some(slot) = store.slots().iter().find(|s| s.name() == &slot_name) else { + continue; + }; + + let storage_map = match slot.content() { + StorageSlotContent::Map(map) => map, + StorageSlotContent::Value(_) => { + // TODO: what to do with value entries? Is it ok to ignore them? + return Err(AccountError::StorageSlotNotMap(slot_name).into()); + }, }; + let details = AccountStorageMapDetails::new(slot_name, slot_data, storage_map); storage_map_details.push(details); } - // Only include unknown account code blobs, which is equal to a account code digest - // mismatch. If `None` was requested, don't return any. - let account_code = code_commitment - .is_some_and(|code_commitment| code_commitment != account.code().commitment()) - .then(|| account.code().to_bytes()); - - // storage details - let storage_details = AccountStorageDetails { - header: storage_header, - map_details: storage_map_details, - }; - - // Handle vault details based on the `asset_vault_commitment`. - // Similar to `code_commitment`, if the provided commitment matches, we don't return - // vault data. If no commitment is provided or it doesn't match, we return - // the vault data. If the number of vault contained assets are exceeding a - // limit, we signal this back in the response and the user must handle that - // in follow-up request. - let vault_details = match asset_vault_commitment { - Some(commitment) if commitment == account.vault().root() => { - // The client already has the correct vault data - AccountVaultDetails::empty() - }, - Some(_) => { - // The commitment doesn't match, so return vault data - AccountVaultDetails::new(account.vault()) - }, - None => { - // No commitment provided, so don't return vault data - AccountVaultDetails::empty() - }, - }; - Ok(AccountDetails { - account_header: AccountHeader::from(account), + account_header, account_code, vault_details, - storage_details, + storage_details: AccountStorageDetails { + header: storage_header, + map_details: storage_map_details, + }, }) } @@ -1076,6 +1086,26 @@ impl State { self.inner.read().await.latest_block_num() } + /// Validates that a block exists in the blockchain + /// + /// # Attention + /// + /// Acquires a *read lock** on `self.inner`. + /// + /// # Errors + /// + /// Returns `DatabaseError::BlockNotFound` if the block doesn't exist in the blockchain. + async fn validate_block_exists(&self, block_num: BlockNumber) -> Result<(), DatabaseError> { + let inner = self.inner.read().await; + let latest_block_num = inner.latest_block_num(); + + if block_num > latest_block_num { + return Err(DatabaseError::BlockNotFound(block_num)); + } + + Ok(()) + } + /// Runs database optimization. pub async fn optimize_db(&self) -> Result<(), DatabaseError> { self.db.optimize().await @@ -1090,6 +1120,18 @@ impl State { self.db.get_account_vault_sync(account_id, block_range).await } + /// Returns the unprocessed network notes, along with the next pagination token. + pub async fn get_unconsumed_network_notes( + &self, + network_account_id_prefix: NetworkAccountPrefix, + block_num: BlockNumber, + page: Page, + ) -> Result<(Vec, Page), DatabaseError> { + self.db + .select_unconsumed_network_notes(network_account_id_prefix, block_num, page) + .await + } + /// Returns the network notes for an account that are unconsumed by a specified block number, /// along with the next pagination token. pub async fn get_unconsumed_network_notes_for_account( @@ -1122,9 +1164,25 @@ impl State { } } -// UTILITIES +// INNER STATE LOADING // ================================================================================================ +#[instrument(level = "info", target = COMPONENT, skip_all)] +async fn load_mmr(db: &mut Db) -> Result { + let block_commitments: Vec = db + .select_all_block_headers() + .await? + .iter() + .map(BlockHeader::commitment) + .collect(); + + // SAFETY: We assume the loaded MMR is valid and does not have more than u32::MAX + // entries. + let chain_mmr = Blockchain::from_mmr_unchecked(block_commitments.into()); + + Ok(chain_mmr) +} + #[instrument(level = "info", target = COMPONENT, skip_all)] async fn load_nullifier_tree( db: &mut Db, @@ -1139,24 +1197,12 @@ async fn load_nullifier_tree( .map_err(StateInitializationError::FailedToCreateNullifierTree) } -#[instrument(level = "info", target = COMPONENT, skip_all)] -async fn load_mmr(db: &mut Db) -> Result { - let block_commitments: Vec = db - .select_all_block_headers() - .await? - .iter() - .map(BlockHeader::commitment) - .collect(); - - Ok(block_commitments.into()) -} - #[instrument(level = "info", target = COMPONENT, skip_all)] async fn load_account_tree( db: &mut Db, block_number: BlockNumber, ) -> Result, StateInitializationError> { - let account_data = db.select_all_account_commitments().await?.into_iter().collect::>(); + let account_data = Vec::from_iter(db.select_all_account_commitments().await?); let smt_entries = account_data .into_iter() From 3346d9f4584b46a9084a2ea523abc1dc5b7aee97 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 30 Dec 2025 00:03:37 +0100 Subject: [PATCH 80/95] cleanup --- crates/store/src/state.rs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 242c2401d5..4431c33e29 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -1026,7 +1026,7 @@ impl State { .db .select_account_header_at_block(account_id, block_num) .await? - .ok_or_else(|| DatabaseError::AccountNotPublic(account_id))?; + .ok_or(DatabaseError::AccountAtBlockHeightNotFoundInDb(account_id, block_num))?; let account_code = match code_commitment { Some(commitment) if commitment == account_header.code_commitment() => None, @@ -1138,19 +1138,6 @@ impl State { ) -> Result<(BlockNumber, Vec), DatabaseError> { self.db.get_account_vault_sync(account_id, block_range).await } - - /// Returns the unprocessed network notes, along with the next pagination token. - pub async fn get_unconsumed_network_notes( - &self, - network_account_id_prefix: NetworkAccountPrefix, - block_num: BlockNumber, - page: Page, - ) -> Result<(Vec, Page), DatabaseError> { - self.db - .select_unconsumed_network_notes(network_account_id_prefix, block_num, page) - .await - } - /// Returns the network notes for an account that are unconsumed by a specified block number, /// along with the next pagination token. pub async fn get_unconsumed_network_notes_for_account( From dbbc1eb3af5ae8c63769ad50d63739e18850dfdf Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 30 Dec 2025 00:11:02 +0100 Subject: [PATCH 81/95] fmt --- crates/store/src/state.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 4431c33e29..6eb422a833 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -1022,11 +1022,11 @@ impl State { // Validate block exists in the blockchain before querying the database self.validate_block_exists(block_num).await?; - let account_header = self - .db - .select_account_header_at_block(account_id, block_num) - .await? - .ok_or(DatabaseError::AccountAtBlockHeightNotFoundInDb(account_id, block_num))?; + let account_header = + self.db + .select_account_header_at_block(account_id, block_num) + .await? + .ok_or(DatabaseError::AccountAtBlockHeightNotFoundInDb(account_id, block_num))?; let account_code = match code_commitment { Some(commitment) if commitment == account_header.code_commitment() => None, From b3c91dfd59197662d02c9de3e914ab8182f40aa8 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 30 Dec 2025 00:22:22 +0100 Subject: [PATCH 82/95] remove dead code --- crates/proto/src/domain/account.rs | 20 +- .../store/src/db/models/queries/accounts.rs | 5 +- .../db/models/queries/accounts/at_block.rs | 5 +- .../src/db/models/queries/accounts/tests.rs | 379 +++++++++++------- crates/store/src/state.rs | 4 +- 5 files changed, 235 insertions(+), 178 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 4c11a44789..1d69c11d0e 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -1,6 +1,7 @@ use std::fmt::{Debug, Display, Formatter}; use miden_node_utils::formatting::format_opt; +use miden_protocol::Word; use miden_protocol::account::{ Account, AccountHeader, @@ -17,7 +18,6 @@ use miden_protocol::block::account_tree::AccountWitness; use miden_protocol::crypto::merkle::SparseMerklePath; use miden_protocol::note::{NoteExecutionMode, NoteTag}; use miden_protocol::utils::{Deserializable, DeserializationError, Serializable}; -use miden_protocol::{AssetError, Word}; use thiserror::Error; use super::try_convert; @@ -389,24 +389,6 @@ impl AccountVaultDetails { Self::Assets(assets) } } - - /// Creates `AccountVaultDetails` from vault entries (key-value pairs). - /// - /// This is useful when entries have been fetched directly from the database - /// rather than extracted from an `AssetVault`. - /// - /// The entries are `(vault_key, asset)` pairs where `asset` is a Word representation. - pub fn from_entries(entries: Vec<(Word, Word)>) -> Result { - if entries.len() > Self::MAX_RETURN_ENTRIES { - return Ok(Self::LimitExceeded); - } - - let assets = Result::, _>::from_iter( - entries.into_iter().map(|(_key, asset_word)| Asset::try_from(asset_word)), - )?; - - Ok(Self::Assets(assets)) - } } impl TryFrom for AccountVaultDetails { diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 290b5d7499..6f7fa10a36 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -56,6 +56,9 @@ pub(crate) use at_block::{ select_account_vault_at_block, }; +#[cfg(test)] +mod tests; + type StorageMapValueRow = (i64, String, Vec, Vec); // ACCOUNT RETRIEVAL @@ -448,7 +451,7 @@ pub(crate) fn select_all_accounts( .order_by(schema::accounts::block_num.asc()) .load::(conn)?; - let summaries: Vec = vec_raw_try_into(raw).unwrap(); + let summaries: Vec = vec_raw_try_into(raw)?; // Backfill account details from database let account_infos = Vec::from_iter(summaries.into_iter().map(|summary| { diff --git a/crates/store/src/db/models/queries/accounts/at_block.rs b/crates/store/src/db/models/queries/accounts/at_block.rs index aaef34a159..882e5b7a94 100644 --- a/crates/store/src/db/models/queries/accounts/at_block.rs +++ b/crates/store/src/db/models/queries/accounts/at_block.rs @@ -279,12 +279,11 @@ pub(crate) fn select_account_storage_at_block( DatabaseError::DataCorrupted(format!("Invalid slot name: {slot_name_str}")) })?; let key = Word::read_from_bytes(&key_bytes)?; + let value = Word::read_from_bytes(&value_bytes)?; // Only insert if we haven't seen this (slot_name, key) yet // (since results are ordered by block_num desc, first one is latest) - latest_map_entries - .entry((slot_name, key)) - .or_insert_with(|| Word::read_from_bytes(&value_bytes).unwrap_or_default()); + latest_map_entries.entry((slot_name, key)).or_insert(value); } // Group entries by slot name diff --git a/crates/store/src/db/models/queries/accounts/tests.rs b/crates/store/src/db/models/queries/accounts/tests.rs index b68df7367e..6f9f5b0755 100644 --- a/crates/store/src/db/models/queries/accounts/tests.rs +++ b/crates/store/src/db/models/queries/accounts/tests.rs @@ -1,19 +1,29 @@ -use assert_matches::assert_matches; -use diesel::{Connection, RunQueryDsl}; +//! Tests for the `accounts` module, specifically for account storage and historical queries. + +use diesel::query_dsl::methods::SelectDsl; +use diesel::{Connection, OptionalExtension, QueryDsl, RunQueryDsl}; use diesel_migrations::MigrationHarness; -use miden_lib::account::auth::AuthRpoFalcon512; -use miden_lib::transaction::TransactionKernel; use miden_node_utils::fee::test_fee_params; -use miden_objects::account::auth::PublicKeyCommitment; -use miden_objects::account::{ +use miden_protocol::account::auth::PublicKeyCommitment; +use miden_protocol::account::delta::AccountUpdateDetails; +use miden_protocol::account::{ + Account, AccountBuilder, AccountComponent, + AccountDelta, + AccountId, AccountIdVersion, AccountStorageMode, AccountType, StorageSlot, + StorageSlotName, }; -use miden_objects::{EMPTY_WORD, Word}; +use miden_protocol::block::{BlockAccountUpdate, BlockHeader, BlockNumber}; +use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SecretKey; +use miden_protocol::utils::Serializable; +use miden_protocol::{EMPTY_WORD, Felt, Word}; +use miden_standards::account::auth::AuthRpoFalcon512; +use miden_standards::code_builder::CodeBuilder; use super::*; use crate::db::migrations::MIGRATIONS; @@ -37,15 +47,15 @@ fn create_test_account_with_storage() -> (Account, AccountId) { ); let storage_value = Word::from([Felt::new(1), Felt::new(2), Felt::new(3), Felt::new(4)]); - let component_storage = vec![StorageSlot::Value(storage_value)]; + let component_storage = vec![StorageSlot::with_value(StorageSlotName::mock(0), storage_value)]; - let component = AccountComponent::compile( - "export.foo push.1 end", - TransactionKernel::assembler(), - component_storage, - ) - .unwrap() - .with_supported_type(AccountType::RegularAccountImmutableCode); + let account_component_code = CodeBuilder::default() + .compile_component_code("test::interface", "pub proc foo push.1 end") + .unwrap(); + + let component = AccountComponent::new(account_component_code, component_storage) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); let account = AccountBuilder::new([1u8; 32]) .account_type(AccountType::RegularAccountImmutableCode) @@ -59,8 +69,6 @@ fn create_test_account_with_storage() -> (Account, AccountId) { } fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { - use miden_objects::block::BlockHeader; - use crate::db::schema::block_headers; let block_header = BlockHeader::new( @@ -73,7 +81,7 @@ fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { Word::default(), Word::default(), Word::default(), - Word::default(), + SecretKey::new().public_key(), test_fee_params(), 0_u8.into(), ); @@ -87,6 +95,163 @@ fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { .expect("Failed to insert block header"); } +// ACCOUNT HEADER AT BLOCK TESTS +// ================================================================================================ + +#[test] +fn test_select_account_header_at_block_returns_none_for_nonexistent() { + let mut conn = setup_test_db(); + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + let account_id = AccountId::dummy( + [99u8; 15], + AccountIdVersion::Version0, + AccountType::RegularAccountImmutableCode, + AccountStorageMode::Public, + ); + + // Query for a non-existent account + let result = select_account_header_at_block(&mut conn, account_id, block_num) + .expect("Query should succeed"); + + assert!(result.is_none(), "Should return None for non-existent account"); +} + +#[test] +fn test_select_account_header_at_block_returns_correct_header() { + let mut conn = setup_test_db(); + let (account, _) = create_test_account_with_storage(); + let account_id = account.id(); + + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + // Insert the account + let delta = AccountDelta::try_from(account.clone()).unwrap(); + let account_update = BlockAccountUpdate::new( + account_id, + account.commitment(), + AccountUpdateDetails::Delta(delta), + ); + + upsert_accounts(&mut conn, &[account_update], block_num).expect("upsert_accounts failed"); + + // Query the account header + let header = select_account_header_at_block(&mut conn, account_id, block_num) + .expect("Query should succeed") + .expect("Header should exist"); + + assert_eq!(header.id(), account_id, "Account ID should match"); + assert_eq!(header.nonce(), account.nonce(), "Nonce should match"); + assert_eq!( + header.code_commitment(), + account.code().commitment(), + "Code commitment should match" + ); +} + +#[test] +fn test_select_account_header_at_block_historical_query() { + let mut conn = setup_test_db(); + let (account, _) = create_test_account_with_storage(); + let account_id = account.id(); + + let block_num_1 = BlockNumber::from_epoch(0); + let block_num_2 = BlockNumber::from_epoch(1); + insert_block_header(&mut conn, block_num_1); + insert_block_header(&mut conn, block_num_2); + + // Insert the account at block 1 + let nonce_1 = account.nonce(); + let delta_1 = AccountDelta::try_from(account.clone()).unwrap(); + let account_update_1 = BlockAccountUpdate::new( + account_id, + account.commitment(), + AccountUpdateDetails::Delta(delta_1), + ); + + upsert_accounts(&mut conn, &[account_update_1], block_num_1).expect("First upsert failed"); + + // Query at block 1 - should return the account + let header_1 = select_account_header_at_block(&mut conn, account_id, block_num_1) + .expect("Query should succeed") + .expect("Header should exist at block 1"); + + assert_eq!(header_1.nonce(), nonce_1, "Nonce at block 1 should match"); + + // Query at block 2 - should return the same account (most recent before block 2) + let header_2 = select_account_header_at_block(&mut conn, account_id, block_num_2) + .expect("Query should succeed") + .expect("Header should exist at block 2"); + + assert_eq!(header_2.nonce(), nonce_1, "Nonce at block 2 should match block 1"); +} + +// ACCOUNT VAULT AT BLOCK TESTS +// ================================================================================================ + +#[test] +fn test_select_account_vault_at_block_empty() { + let mut conn = setup_test_db(); + let (account, _) = create_test_account_with_storage(); + let account_id = account.id(); + + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + // Insert account without vault assets + let delta = AccountDelta::try_from(account.clone()).unwrap(); + let account_update = BlockAccountUpdate::new( + account_id, + account.commitment(), + AccountUpdateDetails::Delta(delta), + ); + + upsert_accounts(&mut conn, &[account_update], block_num).expect("upsert_accounts failed"); + + // Query vault - should return empty (the test account has no assets) + let assets = select_account_vault_at_block(&mut conn, account_id, block_num) + .expect("Query should succeed"); + + assert!(assets.is_empty(), "Account should have no assets"); +} + +// ACCOUNT STORAGE AT BLOCK TESTS +// ================================================================================================ + +#[test] +fn test_select_account_storage_at_block_returns_storage() { + let mut conn = setup_test_db(); + let (account, _) = create_test_account_with_storage(); + let account_id = account.id(); + + let block_num = BlockNumber::from_epoch(0); + insert_block_header(&mut conn, block_num); + + let original_storage_commitment = account.storage().to_commitment(); + + // Insert the account + let delta = AccountDelta::try_from(account.clone()).unwrap(); + let account_update = BlockAccountUpdate::new( + account_id, + account.commitment(), + AccountUpdateDetails::Delta(delta), + ); + + upsert_accounts(&mut conn, &[account_update], block_num).expect("upsert_accounts failed"); + + // Query storage + let storage = select_account_storage_at_block(&mut conn, account_id, block_num) + .expect("Query should succeed"); + + assert_eq!( + storage.to_commitment(), + original_storage_commitment, + "Storage commitment should match" + ); +} + #[test] fn test_upsert_accounts_inserts_storage_header() { let mut conn = setup_test_db(); @@ -96,7 +261,7 @@ fn test_upsert_accounts_inserts_storage_header() { let block_num = BlockNumber::from_epoch(0); insert_block_header(&mut conn, block_num); - let storage_commitment_original = account.storage().commitment(); + let storage_commitment_original = account.storage().to_commitment(); let storage_slots_len = account.storage().slots().len(); let account_commitment = account.commitment(); @@ -118,7 +283,7 @@ fn test_upsert_accounts_inserts_storage_header() { // Verify storage commitment matches assert_eq!( - queried_storage.commitment(), + queried_storage.to_commitment(), storage_commitment_original, "Storage commitment mismatch" ); @@ -151,7 +316,7 @@ fn test_upsert_accounts_updates_is_latest_flag() { insert_block_header(&mut conn, block_num_2); // Save storage commitment before moving account - let storage_commitment_1 = account.storage().commitment(); + let storage_commitment_1 = account.storage().to_commitment(); let account_commitment_1 = account.commitment(); // First update with original account - full state delta @@ -168,15 +333,16 @@ fn test_upsert_accounts_updates_is_latest_flag() { // Create modified account with different storage value let storage_value_modified = Word::from([Felt::new(10), Felt::new(20), Felt::new(30), Felt::new(40)]); - let component_storage_modified = vec![StorageSlot::Value(storage_value_modified)]; + let component_storage_modified = + vec![StorageSlot::with_value(StorageSlotName::mock(0), storage_value_modified)]; - let component_2 = AccountComponent::compile( - "export.foo push.1 end", - TransactionKernel::assembler(), - component_storage_modified, - ) - .unwrap() - .with_supported_type(AccountType::RegularAccountImmutableCode); + let account_component_code = CodeBuilder::default() + .compile_component_code("test::interface", "pub proc foo push.1 end") + .unwrap(); + + let component_2 = AccountComponent::new(account_component_code, component_storage_modified) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); let account_2 = AccountBuilder::new([1u8; 32]) .account_type(AccountType::RegularAccountImmutableCode) @@ -186,7 +352,7 @@ fn test_upsert_accounts_updates_is_latest_flag() { .build_existing() .unwrap(); - let storage_commitment_2 = account_2.storage().commitment(); + let storage_commitment_2 = account_2.storage().to_commitment(); let account_commitment_2 = account_2.commitment(); // Second update with modified account - full state delta @@ -224,7 +390,7 @@ fn test_upsert_accounts_updates_is_latest_flag() { .expect("Failed to query latest storage"); assert_eq!( - latest_storage.commitment(), + latest_storage.to_commitment(), storage_commitment_2, "Latest storage should match second update" ); @@ -234,95 +400,12 @@ fn test_upsert_accounts_updates_is_latest_flag() { .expect("Failed to query storage at block 1"); assert_eq!( - storage_at_block_1.commitment(), + storage_at_block_1.to_commitment(), storage_commitment_1, "Storage at block 1 should match first update" ); } -#[test] -fn test_upsert_accounts_with_incremental_delta() { - use std::collections::BTreeMap; - - use miden_objects::account::delta::{AccountStorageDelta, AccountVaultDelta}; - - let mut conn = setup_test_db(); - let (account, account_id) = create_test_account_with_storage(); - - let block_num_1 = BlockNumber::from_epoch(0); - let block_num_2 = BlockNumber::from_epoch(1); - - insert_block_header(&mut conn, block_num_1); - insert_block_header(&mut conn, block_num_2); - - // First update with full state - let storage_commitment_1 = account.storage().commitment(); - let account_commitment_1 = account.commitment(); - let nonce_1 = account.nonce(); - let delta_1 = AccountDelta::try_from(account).unwrap(); - - let account_update_1 = BlockAccountUpdate::new( - account_id, - account_commitment_1, - AccountUpdateDetails::Delta(delta_1), - ); - - upsert_accounts(&mut conn, &[account_update_1], block_num_1).expect("First upsert failed"); - - // Create incremental delta (only modify storage value slot 1) - let new_storage_value = - Word::from([Felt::new(100), Felt::new(200), Felt::new(300), Felt::new(400)]); - - let mut storage_delta_values = BTreeMap::new(); - storage_delta_values.insert(1u8, new_storage_value); // Update slot 1 (component storage) - - let storage_delta = AccountStorageDelta::from_parts(storage_delta_values, BTreeMap::new()) - .expect("Failed to create storage delta"); - let incremental_delta = - AccountDelta::new(account_id, storage_delta, AccountVaultDelta::default(), nonce_1) - .expect("Failed to create incremental delta"); - - // Reconstruct expected account after delta - let account_after = reconstruct_full_account_from_db(&mut conn, account_id) - .expect("Failed to reconstruct account"); - let mut expected_account = account_after.clone(); - expected_account - .apply_delta(&incremental_delta) - .expect("Failed to apply delta to expected account"); - - let storage_commitment_2 = expected_account.storage().commitment(); - let account_commitment_2 = expected_account.commitment(); - - let account_update_2 = BlockAccountUpdate::new( - account_id, - account_commitment_2, - AccountUpdateDetails::Delta(incremental_delta), - ); - - upsert_accounts(&mut conn, &[account_update_2], block_num_2) - .expect("Second upsert with incremental delta failed"); - - // Verify latest storage matches expected state - let latest_storage = select_latest_account_storage(&mut conn, account_id) - .expect("Failed to query latest storage"); - - assert_eq!( - latest_storage.commitment(), - storage_commitment_2, - "Storage commitment should match after incremental delta" - ); - - // Verify historical storage is preserved - let storage_at_block_1 = select_account_storage_at_block(&mut conn, account_id, block_num_1) - .expect("Failed to query storage at block 1"); - - assert_eq!( - storage_at_block_1.commitment(), - storage_commitment_1, - "Historical storage should be unchanged" - ); -} - #[test] fn test_upsert_accounts_with_multiple_storage_slots() { let mut conn = setup_test_db(); @@ -340,18 +423,18 @@ fn test_upsert_accounts_with_multiple_storage_slots() { let slot_value_3 = Word::from([Felt::new(9), Felt::new(10), Felt::new(11), Felt::new(12)]); let component_storage = vec![ - StorageSlot::Value(slot_value_1), - StorageSlot::Value(slot_value_2), - StorageSlot::Value(slot_value_3), + StorageSlot::with_value(StorageSlotName::mock(0), slot_value_1), + StorageSlot::with_value(StorageSlotName::mock(1), slot_value_2), + StorageSlot::with_value(StorageSlotName::mock(2), slot_value_3), ]; - let component = AccountComponent::compile( - "export.foo push.1 end", - TransactionKernel::assembler(), - component_storage, - ) - .unwrap() - .with_supported_type(AccountType::RegularAccountImmutableCode); + let account_component_code = CodeBuilder::default() + .compile_component_code("test::interface", "pub proc foo push.1 end") + .unwrap(); + + let component = AccountComponent::new(account_component_code, component_storage) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); let account = AccountBuilder::new([2u8; 32]) .account_type(AccountType::RegularAccountImmutableCode) @@ -364,7 +447,7 @@ fn test_upsert_accounts_with_multiple_storage_slots() { let block_num = BlockNumber::from_epoch(0); insert_block_header(&mut conn, block_num); - let storage_commitment = account.storage().commitment(); + let storage_commitment = account.storage().to_commitment(); let account_commitment = account.commitment(); let delta = AccountDelta::try_from(account).unwrap(); @@ -378,7 +461,11 @@ fn test_upsert_accounts_with_multiple_storage_slots() { let queried_storage = select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); - assert_eq!(queried_storage.commitment(), storage_commitment, "Storage commitment mismatch"); + assert_eq!( + queried_storage.to_commitment(), + storage_commitment, + "Storage commitment mismatch" + ); // Note: Auth component adds 1 storage slot, so 3 component slots + 1 auth = 4 total assert_eq!( @@ -387,29 +474,15 @@ fn test_upsert_accounts_with_multiple_storage_slots() { "Expected 4 storage slots (3 component + 1 auth)" ); - // Verify individual slot values (skipping auth slot at index 0) - assert_matches!( - queried_storage.slots().get(1).expect("Slot 1 should exist"), - &StorageSlot::Value(v) if v == slot_value_1, - "Slot 1 value mismatch" - ); - assert_matches!( - queried_storage.slots().get(2).expect("Slot 2 should exist"), - &StorageSlot::Value(v) if v == slot_value_2, - "Slot 2 value mismatch" - ); - assert_matches!( - queried_storage.slots().get(3).expect("Slot 3 should exist"), - &StorageSlot::Value(v) if v == slot_value_3, - "Slot 3 value mismatch" - ); + // The storage commitment matching proves that all values are correctly preserved. + // We don't check individual slot values by index since slot ordering may vary. } #[test] fn test_upsert_accounts_with_empty_storage() { let mut conn = setup_test_db(); - // Create account with no storage slots + // Create account with no component storage slots (only auth slot) let account_id = AccountId::dummy( [3u8; 15], AccountIdVersion::Version0, @@ -417,13 +490,13 @@ fn test_upsert_accounts_with_empty_storage() { AccountStorageMode::Public, ); - let component = AccountComponent::compile( - "export.foo push.1 end", - TransactionKernel::assembler(), - vec![], // Empty storage - ) - .unwrap() - .with_supported_type(AccountType::RegularAccountImmutableCode); + let account_component_code = CodeBuilder::default() + .compile_component_code("test::interface", "pub proc foo push.1 end") + .unwrap(); + + let component = AccountComponent::new(account_component_code, vec![]) + .unwrap() + .with_supported_type(AccountType::RegularAccountImmutableCode); let account = AccountBuilder::new([3u8; 32]) .account_type(AccountType::RegularAccountImmutableCode) @@ -436,7 +509,7 @@ fn test_upsert_accounts_with_empty_storage() { let block_num = BlockNumber::from_epoch(0); insert_block_header(&mut conn, block_num); - let storage_commitment = account.storage().commitment(); + let storage_commitment = account.storage().to_commitment(); let account_commitment = account.commitment(); let delta = AccountDelta::try_from(account).unwrap(); @@ -451,7 +524,7 @@ fn test_upsert_accounts_with_empty_storage() { select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); assert_eq!( - queried_storage.commitment(), + queried_storage.to_commitment(), storage_commitment, "Storage commitment mismatch for empty storage" ); diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 45c4049ec4..5900a697df 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -1019,12 +1019,12 @@ impl State { Some(commitment) if commitment == account_header.vault_root() => { AccountVaultDetails::empty() }, - Some(_) | None if asset_vault_commitment.is_some() => { + Some(_) => { let vault_assets = self.db.select_account_vault_at_block(account_id, block_num).await?; AccountVaultDetails::from_assets(vault_assets) }, - _ => AccountVaultDetails::empty(), + None => AccountVaultDetails::empty(), }; // TODO: don't load the entire store at once, load what is required From 77443c2e0a669b5f67bc2b7fae2ca55035756ead Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 30 Dec 2025 00:35:39 +0100 Subject: [PATCH 83/95] foo --- crates/proto/src/domain/account.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 1d69c11d0e..4d0ec57361 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -71,8 +71,6 @@ impl From for proto::account::AccountId { // ACCOUNT UPDATE // ================================================================================================ - -// TODO should be called `AccountStateRef` or so #[derive(Debug, PartialEq)] pub struct AccountSummary { pub account_id: AccountId, From 1efa4f085a7f7b6e747163a1b00472ac2abf1247 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 30 Dec 2025 00:45:29 +0100 Subject: [PATCH 84/95] fixup --- crates/store/src/db/mod.rs | 7 ++++--- .../store/src/db/models/queries/accounts.rs | 13 +++++++----- crates/store/src/state.rs | 21 +------------------ 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 9f88f0090a..19dac2d1e1 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -112,7 +112,8 @@ impl TransactionRecord { self, note_records: Vec, ) -> proto::rpc::TransactionRecord { - let output_notes = Vec::from_iter(note_records.into_iter().map(Into::into)); + let output_notes: Vec = + note_records.into_iter().map(Into::into).collect(); proto::rpc::TransactionRecord { header: Some(proto::transaction::TransactionHeader { @@ -322,7 +323,7 @@ impl Db { /// Loads all the nullifiers from the DB. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] - pub(crate) async fn select_all_nullifiers(&self) -> Result> { + pub async fn select_all_nullifiers(&self) -> Result> { self.transact("all nullifiers", move |conn| { let nullifiers = queries::select_all_nullifiers(conn)?; Ok(nullifiers) @@ -592,7 +593,7 @@ impl Db { .await } - /// Selects storage map values for syncing storage maps for a specific account ID + /// Selects storage map values for syncing storage maps for a specific account ID. /// /// The returned values are the latest known values up to `block_range.end()`, and no values /// earlier than `block_range.start()` are returned. diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 6f7fa10a36..97e6f28c3a 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -454,11 +454,14 @@ pub(crate) fn select_all_accounts( let summaries: Vec = vec_raw_try_into(raw)?; // Backfill account details from database - let account_infos = Vec::from_iter(summaries.into_iter().map(|summary| { - let account_id = summary.account_id; - let details = select_full_account(conn, account_id).ok(); - AccountInfo { summary, details } - })); + let account_infos = summaries + .into_iter() + .map(|summary| { + let account_id = summary.account_id; + let details = select_full_account(conn, account_id).ok(); + AccountInfo { summary, details } + }) + .collect(); Ok(account_infos) } diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 5900a697df..f36d0ec562 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -103,13 +103,7 @@ where // CHAIN STATE // ================================================================================================ -/// The chain state. -/// -/// The chain state consists of three main components: -/// - A persistent database that stores notes, nullifiers, recent account states, and related data. -/// - In-memory data structures contain Merkle paths for various objects - e.g., all accounts, -/// nullifiers, public account vaults and storage, MMR of all block headers. -/// - Raw block data for all blocks that is stored on disk as flat files. +/// The rollup state. pub struct State { /// The database which stores block headers, nullifiers, notes, and the latest states of /// accounts. @@ -1041,7 +1035,6 @@ impl State { let storage_map = match slot.content() { StorageSlotContent::Map(map) => map, StorageSlotContent::Value(_) => { - // TODO: what to do with value entries? Is it ok to ignore them? return Err(AccountError::StorageSlotNotMap(slot_name).into()); }, }; @@ -1120,18 +1113,6 @@ impl State { self.db.get_account_vault_sync(account_id, block_range).await } - /// Returns the unprocessed network notes, along with the next pagination token. - pub async fn get_unconsumed_network_notes( - &self, - network_account_id_prefix: NetworkAccountPrefix, - block_num: BlockNumber, - page: Page, - ) -> Result<(Vec, Page), DatabaseError> { - self.db - .select_unconsumed_network_notes(network_account_id_prefix, block_num, page) - .await - } - /// Returns the network notes for an account that are unconsumed by a specified block number, /// along with the next pagination token. pub async fn get_unconsumed_network_notes_for_account( From 69ee5a524cc36faf57c17ee189352fb1ba1c64b9 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 30 Dec 2025 00:52:27 +0100 Subject: [PATCH 85/95] change log --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index befd00df0d..6d297d2318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ - Reduced default block interval from `5s` to `2s` ([#1438](https://github.com/0xMiden/miden-node/pull/1438)). - Increased retained account tree history from 33 to 100 blocks to account for the reduced block interval ([#1438](https://github.com/0xMiden/miden-node/pull/1438)). - [BREAKING] Migrated to version `v0.20` of the VM ([#1476](https://github.com/0xMiden/miden-node/pull/1476)). +- [BREAKING] Change account in database representation ([#1481](https://github.com/0xMiden/miden-node/pull/1481)). ### Fixes From 8d33f66b44f4b5c65e6b58f6f5dcdd1929cf42a5 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 30 Dec 2025 17:56:26 +0100 Subject: [PATCH 86/95] fix storage_header comment --- crates/store/src/db/migrations/2025062000000_setup/up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/src/db/migrations/2025062000000_setup/up.sql b/crates/store/src/db/migrations/2025062000000_setup/up.sql index 3f7449292d..38745e6101 100644 --- a/crates/store/src/db/migrations/2025062000000_setup/up.sql +++ b/crates/store/src/db/migrations/2025062000000_setup/up.sql @@ -19,7 +19,7 @@ CREATE TABLE accounts ( account_commitment BLOB NOT NULL, code_commitment BLOB, nonce INTEGER, - storage_header BLOB, -- Serialized AccountStorage from miden-objects + storage_header BLOB, -- Serialized AccountStorageHeader from miden-objects vault_root BLOB, -- Vault root commitment is_latest BOOLEAN NOT NULL DEFAULT 0, -- Indicates if this is the latest state for this account_id From 635cb78e16cd33072d7be06887a899397ec16ae1 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 30 Dec 2025 18:09:05 +0100 Subject: [PATCH 87/95] select_account_code_at_block -> select_account_code_by_commitment --- crates/store/src/db/mod.rs | 13 ++- .../store/src/db/models/queries/accounts.rs | 33 +++++- .../db/models/queries/accounts/at_block.rs | 46 -------- crates/store/src/db/tests.rs | 101 +++++++----------- crates/store/src/state.rs | 6 +- 5 files changed, 79 insertions(+), 120 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 19dac2d1e1..491967eb24 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -467,16 +467,15 @@ impl Db { .await } - /// Queries the account code for a specific account at a specific block number. + /// Queries the account code by its commitment hash. /// - /// Returns `None` if the account doesn't exist at that block or has no code. - pub async fn select_account_code_at_block( + /// Returns `None` if no code exists with that commitment. + pub async fn select_account_code_by_commitment( &self, - account_id: AccountId, - block_num: BlockNumber, + code_commitment: Word, ) -> Result>> { - self.transact("Get account code at block", move |conn| { - queries::select_account_code_at_block(conn, account_id, block_num) + self.transact("Get account code by commitment", move |conn| { + queries::select_account_code_by_commitment(conn, code_commitment) }) .await } diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 97e6f28c3a..3d2f66b05a 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -50,7 +50,6 @@ use crate::errors::DatabaseError; mod at_block; pub(crate) use at_block::{ - select_account_code_at_block, select_account_header_at_block, select_account_storage_at_block, select_account_vault_at_block, @@ -61,6 +60,38 @@ mod tests; type StorageMapValueRow = (i64, String, Vec, Vec); +// ACCOUNT CODE +// ================================================================================================ + +/// Select account code by its commitment hash from the `account_codes` table. +/// +/// # Returns +/// +/// The account code bytes if found, or `None` if no code exists with that commitment. +/// +/// # Raw SQL +/// +/// ```sql +/// SELECT code FROM account_codes WHERE code_commitment = ?1 +/// ``` +pub(crate) fn select_account_code_by_commitment( + conn: &mut SqliteConnection, + code_commitment: Word, +) -> Result>, DatabaseError> { + use schema::account_codes; + + let code_commitment_bytes = code_commitment.to_bytes(); + + let result: Option> = SelectDsl::select( + account_codes::table.filter(account_codes::code_commitment.eq(&code_commitment_bytes)), + account_codes::code, + ) + .first(conn) + .optional()?; + + Ok(result) +} + // ACCOUNT RETRIEVAL // ================================================================================================ diff --git a/crates/store/src/db/models/queries/accounts/at_block.rs b/crates/store/src/db/models/queries/accounts/at_block.rs index 882e5b7a94..021714abe6 100644 --- a/crates/store/src/db/models/queries/accounts/at_block.rs +++ b/crates/store/src/db/models/queries/accounts/at_block.rs @@ -122,52 +122,6 @@ pub(crate) fn select_account_header_at_block( ))) } -// ACCOUNT CODE -// ================================================================================================ - -/// Queries the account code for a specific account at a specific block number. -/// -/// Returns `None` if: -/// - The account doesn't exist at that block -/// - The account has no code (private account or account without code commitment) -/// -/// # Arguments -/// -/// * `conn` - Database connection -/// * `account_id` - The account ID to query -/// * `block_num` - The block number at which to query the account code -/// -/// # Returns -/// -/// * `Ok(Some(Vec))` - The account code bytes if found -/// * `Ok(None)` - If account doesn't exist or has no code -/// * `Err(DatabaseError)` - If there's a database error -pub(crate) fn select_account_code_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result>, DatabaseError> { - use schema::{account_codes, accounts}; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = i64::from(block_num.as_u32()); - // Query the accounts table to get the code_commitment at the specified block or earlier - // Then join with account_codes to get the actual code - let result: Option> = SelectDsl::select( - accounts::table - .inner_join(account_codes::table) - .filter(accounts::account_id.eq(&account_id_bytes)) - .filter(accounts::block_num.le(block_num_sql)) - .order(accounts::block_num.desc()) - .limit(1), - account_codes::code, - ) - .first(conn) - .optional()?; - - Ok(result) -} - // ACCOUNT VAULT // ================================================================================================ diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 488b9232de..6aa25417b4 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1452,11 +1452,11 @@ fn mock_account_code_and_storage( .unwrap() } -// STORAGE RECONSTRUCTION TESTS +// ACCOUNT CODE TESTS // ================================================================================================ #[test] -fn test_select_account_code_at_block() { +fn test_select_account_code_by_commitment() { let mut conn = create_db(); let block_num_1 = BlockNumber::from(1); @@ -1472,17 +1472,15 @@ fn test_select_account_code_at_block() { None, ); - // Use the actual account ID from the created account - let account_id = account.id(); - - // Get the code bytes before inserting + // Get the code commitment and bytes before inserting + let code_commitment = account.code().commitment(); let expected_code = account.code().to_bytes(); // Insert the account at block 1 queries::upsert_accounts( &mut conn, &[BlockAccountUpdate::new( - account_id, + account.id(), account.commitment(), AccountUpdateDetails::Delta(AccountDelta::try_from(account).unwrap()), )], @@ -1490,33 +1488,32 @@ fn test_select_account_code_at_block() { ) .unwrap(); - // Query code at block 1 - should return the code - let code_at_1 = queries::select_account_code_at_block(&mut conn, account_id, block_num_1) + // Query code by commitment - should return the code + let code = queries::select_account_code_by_commitment(&mut conn, code_commitment) .unwrap() - .expect("Code should exist at block 1"); - assert_eq!(code_at_1, expected_code); + .expect("Code should exist"); + assert_eq!(code, expected_code); - // Query code for non-existent account - should return None - let other_account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + // Query code for non-existent commitment - should return None + let non_existent_commitment = [0u8; 32]; + let non_existent_commitment = Word::read_from_bytes(&non_existent_commitment).unwrap(); let code_other = - queries::select_account_code_at_block(&mut conn, other_account_id, block_num_1).unwrap(); - assert!(code_other.is_none(), "Code should not exist for non-existent account"); + queries::select_account_code_by_commitment(&mut conn, non_existent_commitment).unwrap(); + assert!(code_other.is_none(), "Code should not exist for non-existent commitment"); } #[test] -fn test_select_account_code_at_block_with_updates() { +fn test_select_account_code_by_commitment_multiple_codes() { let mut conn = create_db(); let block_num_1 = BlockNumber::from(1); let block_num_2 = BlockNumber::from(2); - let block_num_3 = BlockNumber::from(3); - // Create all blocks + // Create blocks create_block(&mut conn, block_num_1); create_block(&mut conn, block_num_2); - create_block(&mut conn, block_num_3); - // Create initial account with code v1 at block 1 + // Create account with code v1 at block 1 let code_v1_str = "\ pub proc account_procedure_1 push.1.2 @@ -1524,14 +1521,14 @@ fn test_select_account_code_at_block_with_updates() { end "; let account_v1 = create_account_with_code(code_v1_str, [1u8; 32]); - let account_id = account_v1.id(); + let code_v1_commitment = account_v1.code().commitment(); let code_v1 = account_v1.code().to_bytes(); // Insert the account at block 1 queries::upsert_accounts( &mut conn, &[BlockAccountUpdate::new( - account_id, + account_v1.id(), account_v1.commitment(), AccountUpdateDetails::Delta(AccountDelta::try_from(account_v1).unwrap()), )], @@ -1547,6 +1544,7 @@ fn test_select_account_code_at_block_with_updates() { end "; let account_v2 = create_account_with_code(code_v2_str, [1u8; 32]); // Same seed to keep same account_id + let code_v2_commitment = account_v2.code().commitment(); let code_v2 = account_v2.code().to_bytes(); // Verify that the codes are actually different @@ -1554,12 +1552,16 @@ fn test_select_account_code_at_block_with_updates() { code_v1, code_v2, "Test setup error: codes should be different for different code strings" ); + assert_ne!( + code_v1_commitment, code_v2_commitment, + "Test setup error: code commitments should be different" + ); // Insert the updated account at block 2 queries::upsert_accounts( &mut conn, &[BlockAccountUpdate::new( - account_id, + account_v2.id(), account_v2.commitment(), AccountUpdateDetails::Delta(AccountDelta::try_from(account_v2).unwrap()), )], @@ -1567,49 +1569,18 @@ fn test_select_account_code_at_block_with_updates() { ) .unwrap(); - // Create account with different code v3 at block 3 - let code_v3_str = "\ - pub proc account_procedure_1 - push.5.6 - sub - end - "; - let account_v3 = create_account_with_code(code_v3_str, [1u8; 32]); // Same seed to keep same account_id - let code_v3 = account_v3.code().to_bytes(); - - // Verify that v3 code is different from v2 and v1 - assert_ne!(code_v2, code_v3, "Test setup error: v3 code should differ from v2"); - assert_ne!(code_v1, code_v3, "Test setup error: v3 code should differ from v1"); - - // Insert the updated account at block 3 - queries::upsert_accounts( - &mut conn, - &[BlockAccountUpdate::new( - account_id, - account_v3.commitment(), - AccountUpdateDetails::Delta(AccountDelta::try_from(account_v3).unwrap()), - )], - block_num_3, - ) - .unwrap(); - - // Test: Query code at block 1 - should return v1 code - let code_at_1 = queries::select_account_code_at_block(&mut conn, account_id, block_num_1) - .unwrap() - .expect("Code should exist at block 1"); - assert_eq!(code_at_1, code_v1, "Block 1 should return v1 code"); - - // Test: Query code at block 2 - should return v2 code (even though we're at block 3) - let code_at_2 = queries::select_account_code_at_block(&mut conn, account_id, block_num_2) - .unwrap() - .expect("Code should exist at block 2"); - assert_eq!(code_at_2, code_v2, "Block 2 should return v2 code"); + // Both codes should be retrievable by their respective commitments + let code_from_v1_commitment = + queries::select_account_code_by_commitment(&mut conn, code_v1_commitment) + .unwrap() + .expect("v1 code should exist"); + assert_eq!(code_from_v1_commitment, code_v1, "v1 commitment should return v1 code"); - // Test: Query code at block 3 - should return v3 code - let code_at_3 = queries::select_account_code_at_block(&mut conn, account_id, block_num_3) - .unwrap() - .expect("Code should exist at block 3"); - assert_eq!(code_at_3, code_v3, "Block 3 should return v3 code"); + let code_from_v2_commitment = + queries::select_account_code_by_commitment(&mut conn, code_v2_commitment) + .unwrap() + .expect("v2 code should exist"); + assert_eq!(code_from_v2_commitment, code_v2, "v2 commitment should return v2 code"); } // GENESIS REGRESSION TESTS diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index f36d0ec562..7ba7b3a8b1 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -1005,7 +1005,11 @@ impl State { let account_code = match code_commitment { Some(commitment) if commitment == account_header.code_commitment() => None, - Some(_) => self.db.select_account_code_at_block(account_id, block_num).await?, + Some(_) => { + self.db + .select_account_code_by_commitment(account_header.code_commitment()) + .await? + }, None => None, }; From 7ca699981139e3150b36f14882f0b79b367be7a2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 30 Dec 2025 18:19:45 +0100 Subject: [PATCH 88/95] add minor test xtension --- crates/store/src/db/tests.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 6aa25417b4..3988e160da 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -2337,4 +2337,14 @@ fn db_roundtrip_account_storage_with_maps() { _ => unreachable!(), } } + + // Also verify full account reconstruction via select_account (which calls select_full_account) + let account_info = queries::select_account(&mut conn, account_id).unwrap(); + assert!(account_info.details.is_some(), "Public account should have details"); + let retrieved_account = account_info.details.unwrap(); + assert_eq!( + account.commitment(), + retrieved_account.commitment(), + "Full account commitment must match after DB roundtrip" + ); } From d9a666f5cbf7b306c7fd6af4767535330f3385da Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jan 2026 18:54:45 +0100 Subject: [PATCH 89/95] review --- crates/store/src/accounts/mod.rs | 3 --- crates/store/src/db/mod.rs | 2 +- crates/store/src/inner_forest/mod.rs | 36 ++++++++++++++++++++++++---- crates/store/src/lib.rs | 1 - crates/store/src/state.rs | 7 +++--- 5 files changed, 37 insertions(+), 12 deletions(-) diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index 2e680fa947..c0a37be323 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -62,9 +62,7 @@ enum HistoricalSelector { /// Captures reversion state for historical queries at a specific block. #[derive(Debug, Clone)] struct HistoricalOverlay { - #[allow(dead_code)] block_number: BlockNumber, - #[allow(dead_code)] root: Word, node_mutations: HashMap, account_updates: HashMap, (Word, Word)>, @@ -159,7 +157,6 @@ impl AccountTreeWithHistory { /// Returns the root hash at a specific historical block. /// /// Returns `None` if the block is in the future or too old (pruned). - #[cfg(test)] pub fn root_at(&self, block_number: BlockNumber) -> Option { match self.historical_selector(block_number) { HistoricalSelector::Latest => Some(self.latest.root()), diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 53f7c2397d..c2a2124ec9 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -391,7 +391,7 @@ impl Db { .await } - /// TODO marked for removal, replace with paged version + /// TODO marked for removal, replace with paged version. #[instrument(level = "debug", target = COMPONENT, skip_all, ret(level = "debug"), err)] pub async fn select_all_account_commitments(&self) -> Result> { self.transact("read all account commitments", move |conn| { diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 5778091af0..83218ee5ab 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -46,6 +46,18 @@ impl InnerForest { *EmptySubtreeRoots::entry(SMT_DEPTH, 0) } + /// Retrieves the most recent vault SMT root for an account. + /// + /// Returns the latest vault root entry regardless of block number. + /// Used when applying incremental deltas where we always want the previous state. + fn get_latest_vault_root(&self, account_id: AccountId) -> Word { + self.vault_roots + .range((account_id, BlockNumber::GENESIS)..) + .take_while(|((id, _), _)| *id == account_id) + .last() + .map_or_else(Self::empty_smt_root, |(_, root)| *root) + } + /// Retrieves the vault SMT root for an account at or before the given block. /// /// Finds the most recent vault root entry for the account, since vault state persists @@ -53,6 +65,7 @@ impl InnerForest { // // TODO: a fallback to DB lookup is required once pruning lands. // Currently returns empty root which would be incorrect + #[cfg(test)] fn get_vault_root(&self, account_id: AccountId, block_num: BlockNumber) -> Word { self.vault_roots .range((account_id, BlockNumber::GENESIS)..=(account_id, block_num)) @@ -60,6 +73,22 @@ impl InnerForest { .map_or_else(Self::empty_smt_root, |(_, root)| *root) } + /// Retrieves the most recent storage map SMT root for an account slot. + /// + /// Returns the latest storage root entry regardless of block number. + /// Used when applying incremental deltas where we always want the previous state. + fn get_latest_storage_map_root( + &self, + account_id: AccountId, + slot_name: &StorageSlotName, + ) -> Word { + self.storage_roots + .range((account_id, slot_name.clone(), BlockNumber::GENESIS)..) + .take_while(|((id, name, _), _)| *id == account_id && name == slot_name) + .last() + .map_or_else(Self::empty_smt_root, |(_, root)| *root) + } + /// Retrieves the storage map SMT root for an account slot at or before the given block. /// /// Finds the most recent storage root entry for the slot, since storage state persists @@ -67,6 +96,7 @@ impl InnerForest { // // TODO: a fallback to DB lookup is required once pruning lands. // Currently returns empty root which would be incorrect + #[cfg(test)] fn get_storage_root( &self, account_id: AccountId, @@ -150,7 +180,7 @@ impl InnerForest { let prev_root = if is_full_state { Self::empty_smt_root() } else { - self.get_vault_root(account_id, block_num.parent().unwrap_or_default()) + self.get_latest_vault_root(account_id) }; let mut entries = Vec::new(); @@ -231,13 +261,11 @@ impl InnerForest { storage_delta: &AccountStorageDelta, is_full_state: bool, ) { - let parent_block = block_num.parent().unwrap_or_default(); - for (slot_name, map_delta) in storage_delta.maps() { let prev_root = if is_full_state { Self::empty_smt_root() } else { - self.get_storage_root(account_id, slot_name, parent_block) + self.get_latest_storage_map_root(account_id, slot_name) }; let entries: Vec<_> = diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 633464e459..582dfdc2fc 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -9,7 +9,6 @@ mod inner_forest; mod server; pub mod state; -pub(crate) use accounts::{AccountTreeWithHistory, HistoricalError}; pub use genesis::GenesisState; pub use server::{DataDirectory, Store}; diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index c7f935c38d..14b8a7a022 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -43,6 +43,7 @@ use miden_protocol::{AccountError, Word}; use tokio::sync::{Mutex, RwLock, oneshot}; use tracing::{info, info_span, instrument}; +use crate::accounts::{AccountTreeWithHistory, HistoricalError}; use crate::blocks::BlockStore; use crate::db::models::Page; use crate::db::models::queries::StorageMapValuesPage; @@ -67,7 +68,7 @@ use crate::errors::{ StateSyncError, }; use crate::inner_forest::InnerForest; -use crate::{AccountTreeWithHistory, COMPONENT, DataDirectory}; +use crate::{COMPONENT, DataDirectory}; // STRUCTURES // ================================================================================================ @@ -304,10 +305,10 @@ impl State { .map(|update| (update.account_id(), update.final_state_commitment())), ) .map_err(|e| match e { - crate::HistoricalError::AccountTreeError(err) => { + HistoricalError::AccountTreeError(err) => { InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) }, - crate::HistoricalError::MerkleError(_) => { + HistoricalError::MerkleError(_) => { panic!("Unexpected MerkleError during account tree mutation computation") }, })?; From 29b840c91c730a12f1b8245c9f59edc6fb184c7e Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 6 Jan 2026 19:03:52 +0100 Subject: [PATCH 90/95] remove dead code --- crates/store/src/inner_forest/mod.rs | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 83218ee5ab..5ff50d47b1 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -89,29 +89,6 @@ impl InnerForest { .map_or_else(Self::empty_smt_root, |(_, root)| *root) } - /// Retrieves the storage map SMT root for an account slot at or before the given block. - /// - /// Finds the most recent storage root entry for the slot, since storage state persists - /// across blocks where no changes occur. - // - // TODO: a fallback to DB lookup is required once pruning lands. - // Currently returns empty root which would be incorrect - #[cfg(test)] - fn get_storage_root( - &self, - account_id: AccountId, - slot_name: &StorageSlotName, - block_num: BlockNumber, - ) -> Word { - self.storage_roots - .range( - (account_id, slot_name.clone(), BlockNumber::GENESIS) - ..=(account_id, slot_name.clone(), block_num), - ) - .next_back() - .map_or_else(Self::empty_smt_root, |(_, root)| *root) - } - // PUBLIC INTERFACE // -------------------------------------------------------------------------------------------- From 1b22a340807be033142e06b87daa1c2a22fa6b19 Mon Sep 17 00:00:00 2001 From: Bobbin Threadbare Date: Tue, 6 Jan 2026 17:23:13 -0800 Subject: [PATCH 91/95] chore: minor rename --- crates/store/src/db/mod.rs | 2 +- crates/store/src/inner_forest/mod.rs | 8 ++++---- crates/store/src/inner_forest/tests.rs | 18 +++++++++++------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index c2a2124ec9..b01964cbee 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -590,7 +590,7 @@ impl Db { .await } - /// Selects storage map values for syncing storage maps for a specific account ID + /// Selects storage map values for syncing storage maps for a specific account ID. /// /// The returned values are the latest known values up to `block_range.end()`, and no values /// earlier than `block_range.start()` are returned. diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 5ff50d47b1..f77167e6e5 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -22,7 +22,7 @@ pub(crate) struct InnerForest { /// Maps (`account_id`, `slot_name`, `block_num`) to SMT root. /// Populated during block import for all storage map slots. - storage_roots: BTreeMap<(AccountId, StorageSlotName, BlockNumber), Word>, + storage_map_roots: BTreeMap<(AccountId, StorageSlotName, BlockNumber), Word>, /// Maps (`account_id`, `block_num`) to vault SMT root. /// Tracks asset vault versions across all blocks with structural sharing. @@ -33,7 +33,7 @@ impl InnerForest { pub(crate) fn new() -> Self { Self { forest: SmtForest::new(), - storage_roots: BTreeMap::new(), + storage_map_roots: BTreeMap::new(), vault_roots: BTreeMap::new(), } } @@ -82,7 +82,7 @@ impl InnerForest { account_id: AccountId, slot_name: &StorageSlotName, ) -> Word { - self.storage_roots + self.storage_map_roots .range((account_id, slot_name.clone(), BlockNumber::GENESIS)..) .take_while(|((id, name, _), _)| *id == account_id && name == slot_name) .last() @@ -257,7 +257,7 @@ impl InnerForest { .batch_insert(prev_root, entries.iter().copied()) .expect("forest insertion should succeed"); - self.storage_roots + self.storage_map_roots .insert((account_id, slot_name.clone(), block_num), updated_root); tracing::debug!( diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs index d7351b892a..5bbd952c3c 100644 --- a/crates/store/src/inner_forest/tests.rs +++ b/crates/store/src/inner_forest/tests.rs @@ -70,7 +70,7 @@ fn test_empty_smt_root_is_recognized() { #[test] fn test_inner_forest_basic_initialization() { let forest = InnerForest::new(); - assert!(forest.storage_roots.is_empty()); + assert!(forest.storage_map_roots.is_empty()); assert!(forest.vault_roots.is_empty()); } @@ -90,7 +90,7 @@ fn test_update_account_with_empty_deltas() { // Empty deltas should not create entries assert!(!forest.vault_roots.contains_key(&(account_id, block_num))); - assert!(forest.storage_roots.is_empty()); + assert!(forest.storage_map_roots.is_empty()); } #[test] @@ -359,8 +359,12 @@ fn test_update_storage_map() { forest.update_account(block_num, &delta); // Verify storage root was created - assert!(forest.storage_roots.contains_key(&(account_id, slot_name.clone(), block_num))); - let storage_root = forest.storage_roots[&(account_id, slot_name, block_num)]; + assert!( + forest + .storage_map_roots + .contains_key(&(account_id, slot_name.clone(), block_num)) + ); + let storage_root = forest.storage_map_roots[&(account_id, slot_name, block_num)]; assert_ne!(storage_root, InnerForest::empty_smt_root()); } @@ -388,7 +392,7 @@ fn test_storage_map_incremental_updates() { let storage_delta_1 = AccountStorageDelta::from_raw(raw_1); let delta_1 = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta_1); forest.update_account(block_1, &delta_1); - let root_1 = forest.storage_roots[&(account_id, slot_name.clone(), block_1)]; + let root_1 = forest.storage_map_roots[&(account_id, slot_name.clone(), block_1)]; // Block 2: Insert key2 -> value2 (key1 should persist) let block_2 = block_1.child(); @@ -398,7 +402,7 @@ fn test_storage_map_incremental_updates() { let storage_delta_2 = AccountStorageDelta::from_raw(raw_2); let delta_2 = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta_2); forest.update_account(block_2, &delta_2); - let root_2 = forest.storage_roots[&(account_id, slot_name.clone(), block_2)]; + let root_2 = forest.storage_map_roots[&(account_id, slot_name.clone(), block_2)]; // Block 3: Update key1 -> value3 let block_3 = block_2.child(); @@ -408,7 +412,7 @@ fn test_storage_map_incremental_updates() { let storage_delta_3 = AccountStorageDelta::from_raw(raw_3); let delta_3 = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta_3); forest.update_account(block_3, &delta_3); - let root_3 = forest.storage_roots[&(account_id, slot_name, block_3)]; + let root_3 = forest.storage_map_roots[&(account_id, slot_name, block_3)]; // All roots should be different assert_ne!(root_1, root_2); From 1cab5e2092a14b02690d70246a6d7b10c019cdf8 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 9 Jan 2026 17:01:20 +0100 Subject: [PATCH 92/95] review --- crates/store/src/inner_forest/mod.rs | 58 +++++++++++++++------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index f77167e6e5..35181ddd3a 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -50,7 +50,12 @@ impl InnerForest { /// /// Returns the latest vault root entry regardless of block number. /// Used when applying incremental deltas where we always want the previous state. - fn get_latest_vault_root(&self, account_id: AccountId) -> Word { + /// + /// If no vault root is found for the account, returns an empty SMT root. + fn get_latest_vault_root(&self, account_id: AccountId, is_full_state: bool) -> Word { + if is_full_state { + return Self::empty_smt_root(); + } self.vault_roots .range((account_id, BlockNumber::GENESIS)..) .take_while(|((id, _), _)| *id == account_id) @@ -58,30 +63,22 @@ impl InnerForest { .map_or_else(Self::empty_smt_root, |(_, root)| *root) } - /// Retrieves the vault SMT root for an account at or before the given block. - /// - /// Finds the most recent vault root entry for the account, since vault state persists - /// across blocks where no changes occur. - // - // TODO: a fallback to DB lookup is required once pruning lands. - // Currently returns empty root which would be incorrect - #[cfg(test)] - fn get_vault_root(&self, account_id: AccountId, block_num: BlockNumber) -> Word { - self.vault_roots - .range((account_id, BlockNumber::GENESIS)..=(account_id, block_num)) - .next_back() - .map_or_else(Self::empty_smt_root, |(_, root)| *root) - } - /// Retrieves the most recent storage map SMT root for an account slot. /// /// Returns the latest storage root entry regardless of block number. /// Used when applying incremental deltas where we always want the previous state. + /// + /// If no storage root is found for the slot, returns an empty SMT root. fn get_latest_storage_map_root( &self, account_id: AccountId, slot_name: &StorageSlotName, + is_full_state: bool, ) -> Word { + if is_full_state { + return Self::empty_smt_root(); + } + self.storage_map_roots .range((account_id, slot_name.clone(), BlockNumber::GENESIS)..) .take_while(|((id, name, _), _)| *id == account_id && name == slot_name) @@ -89,6 +86,21 @@ impl InnerForest { .map_or_else(Self::empty_smt_root, |(_, root)| *root) } + /// Retrieves the vault SMT root for an account at or before the given block. + /// + /// Finds the most recent vault root entry for the account, since vault state persists + /// across blocks where no changes occur. + // + // TODO: a fallback to DB lookup is required once pruning lands. + // Currently returns empty root which would be incorrect + #[cfg(test)] + fn get_vault_root(&self, account_id: AccountId, block_num: BlockNumber) -> Word { + self.vault_roots + .range((account_id, BlockNumber::GENESIS)..=(account_id, block_num)) + .next_back() + .map_or_else(Self::empty_smt_root, |(_, root)| *root) + } + // PUBLIC INTERFACE // -------------------------------------------------------------------------------------------- @@ -154,11 +166,7 @@ impl InnerForest { vault_delta: &AccountVaultDelta, is_full_state: bool, ) { - let prev_root = if is_full_state { - Self::empty_smt_root() - } else { - self.get_latest_vault_root(account_id) - }; + let prev_root = self.get_latest_vault_root(account_id, is_full_state); let mut entries = Vec::new(); @@ -184,7 +192,7 @@ impl InnerForest { .map_or(0, |asset| asset.amount()); let new_balance = i128::from(prev_amount) + i128::from(*amount_delta); - u64::try_from(new_balance.max(0)).expect("balance fits in u64") + u64::try_from(new_balance).expect("balance should be non-negative and fit in u64") }; let value = if new_amount == 0 { @@ -239,11 +247,7 @@ impl InnerForest { is_full_state: bool, ) { for (slot_name, map_delta) in storage_delta.maps() { - let prev_root = if is_full_state { - Self::empty_smt_root() - } else { - self.get_latest_storage_map_root(account_id, slot_name) - }; + let prev_root = self.get_latest_storage_map_root(account_id, slot_name, is_full_state); let entries: Vec<_> = map_delta.entries().iter().map(|(key, value)| ((*key).into(), *value)).collect(); From c42d5f5fd9d90bf9cda4ead4f7da02bddf4127b9 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 9 Jan 2026 17:41:50 +0100 Subject: [PATCH 93/95] some more docs --- crates/store/src/inner_forest/mod.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 35181ddd3a..7a96a2a8f6 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -52,6 +52,12 @@ impl InnerForest { /// Used when applying incremental deltas where we always want the previous state. /// /// If no vault root is found for the account, returns an empty SMT root. + /// + /// # Arguments + /// + /// * `is_full_state` - If `true`, returns an empty SMT root (for new accounts or DB + /// reconstruction where delta values are absolute). If `false`, looks up the previous state + /// (for incremental updates where delta values are relative changes). fn get_latest_vault_root(&self, account_id: AccountId, is_full_state: bool) -> Word { if is_full_state { return Self::empty_smt_root(); @@ -69,6 +75,12 @@ impl InnerForest { /// Used when applying incremental deltas where we always want the previous state. /// /// If no storage root is found for the slot, returns an empty SMT root. + /// + /// # Arguments + /// + /// * `is_full_state` - If `true`, returns an empty SMT root (for new accounts or DB + /// reconstruction where delta values are absolute). If `false`, looks up the previous state + /// (for incremental updates where delta values are relative changes). fn get_latest_storage_map_root( &self, account_id: AccountId, @@ -159,6 +171,11 @@ impl InnerForest { /// /// Processes both fungible and non-fungible asset changes, building entries for the vault SMT /// and tracking the new root. + /// + /// # Arguments + /// + /// * `is_full_state` - If `true`, delta values are absolute (new account or DB reconstruction). + /// If `false`, delta values are relative changes applied to previous state. fn update_account_vault( &mut self, block_num: BlockNumber, @@ -239,6 +256,11 @@ impl InnerForest { /// /// Processes storage map slot deltas, building SMTs for each modified slot /// and tracking the new roots. + /// + /// # Arguments + /// + /// * `is_full_state` - If `true`, delta values are absolute (new account or DB reconstruction). + /// If `false`, delta values are relative changes applied to previous state. fn update_account_storage( &mut self, block_num: BlockNumber, From 9fe3979a7de66f6ee26fa8183f096cb2f1eaa92a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Sat, 10 Jan 2026 14:03:01 +0100 Subject: [PATCH 94/95] add InnerForestError type, make asset addition non-panic --- crates/store/src/errors.rs | 5 +++ crates/store/src/inner_forest/mod.rs | 56 ++++++++++++++++++++++---- crates/store/src/inner_forest/tests.rs | 40 +++++++++--------- crates/store/src/state.rs | 4 +- 4 files changed, 76 insertions(+), 29 deletions(-) diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 7ac836ed3f..7e0c326a23 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -31,6 +31,7 @@ use tonic::Status; use crate::db::manager::ConnectionManagerError; use crate::db::models::conv::DatabaseTypeConversionError; +use crate::inner_forest::InnerForestError; // DATABASE ERRORS // ================================================================================================= @@ -197,6 +198,8 @@ pub enum StateInitializationError { BlockStoreLoadError(#[source] std::io::Error), #[error("failed to load database")] DatabaseLoadError(#[from] DatabaseSetupError), + #[error("inner forest error")] + InnerForestError(#[from] InnerForestError), } #[derive(Debug, Error)] @@ -274,6 +277,8 @@ pub enum ApplyBlockError { TokioJoinError(#[from] tokio::task::JoinError), #[error("invalid block error")] InvalidBlockError(#[from] InvalidBlockError), + #[error("inner forest error")] + InnerForestError(#[from] InnerForestError), // OTHER ERRORS // --------------------------------------------------------------------------------------------- diff --git a/crates/store/src/inner_forest/mod.rs b/crates/store/src/inner_forest/mod.rs index 7a96a2a8f6..d368896f26 100644 --- a/crates/store/src/inner_forest/mod.rs +++ b/crates/store/src/inner_forest/mod.rs @@ -7,10 +7,28 @@ use miden_protocol::block::BlockNumber; use miden_protocol::crypto::merkle::EmptySubtreeRoots; use miden_protocol::crypto::merkle::smt::{SMT_DEPTH, SmtForest}; use miden_protocol::{EMPTY_WORD, Word}; +use thiserror::Error; #[cfg(test)] mod tests; +// ERRORS +// ================================================================================================ + +#[derive(Debug, Error)] +pub enum InnerForestError { + #[error( + "balance underflow: account {account_id}, faucet {faucet_id}, \ + previous balance {prev_balance}, delta {delta}" + )] + BalanceUnderflow { + account_id: AccountId, + faucet_id: AccountId, + prev_balance: u64, + delta: i64, + }, +} + // INNER FOREST // ================================================================================================ @@ -125,13 +143,17 @@ impl InnerForest { /// /// * `block_num` - Block number for which these updates apply /// * `account_updates` - Iterator of `AccountDelta` for public accounts + /// + /// # Errors + /// + /// Returns an error if applying a vault delta results in a negative balance. pub(crate) fn apply_block_updates( &mut self, block_num: BlockNumber, account_updates: impl IntoIterator, - ) { + ) -> Result<(), InnerForestError> { for delta in account_updates { - self.update_account(block_num, &delta); + self.update_account(block_num, &delta)?; tracing::debug!( target: crate::COMPONENT, @@ -141,6 +163,7 @@ impl InnerForest { "Updated forest with account delta" ); } + Ok(()) } /// Updates the forest with account vault and storage changes from a delta. @@ -151,17 +174,26 @@ impl InnerForest { /// /// Full-state deltas (`delta.is_full_state() == true`) populate the forest from scratch using /// an empty SMT root. Partial deltas apply changes on top of the previous block's state. - pub(crate) fn update_account(&mut self, block_num: BlockNumber, delta: &AccountDelta) { + /// + /// # Errors + /// + /// Returns an error if applying a vault delta results in a negative balance. + pub(crate) fn update_account( + &mut self, + block_num: BlockNumber, + delta: &AccountDelta, + ) -> Result<(), InnerForestError> { let account_id = delta.id(); let is_full_state = delta.is_full_state(); if !delta.vault().is_empty() { - self.update_account_vault(block_num, account_id, delta.vault(), is_full_state); + self.update_account_vault(block_num, account_id, delta.vault(), is_full_state)?; } if !delta.storage().is_empty() { self.update_account_storage(block_num, account_id, delta.storage(), is_full_state); } + Ok(()) } // PRIVATE METHODS @@ -176,13 +208,17 @@ impl InnerForest { /// /// * `is_full_state` - If `true`, delta values are absolute (new account or DB reconstruction). /// If `false`, delta values are relative changes applied to previous state. + /// + /// # Errors + /// + /// Returns an error if applying a delta results in a negative balance. fn update_account_vault( &mut self, block_num: BlockNumber, account_id: AccountId, vault_delta: &AccountVaultDelta, is_full_state: bool, - ) { + ) -> Result<(), InnerForestError> { let prev_root = self.get_latest_vault_root(account_id, is_full_state); let mut entries = Vec::new(); @@ -209,7 +245,12 @@ impl InnerForest { .map_or(0, |asset| asset.amount()); let new_balance = i128::from(prev_amount) + i128::from(*amount_delta); - u64::try_from(new_balance).expect("balance should be non-negative and fit in u64") + u64::try_from(new_balance).map_err(|_| InnerForestError::BalanceUnderflow { + account_id, + faucet_id: *faucet_id, + prev_balance: prev_amount, + delta: *amount_delta, + })? }; let value = if new_amount == 0 { @@ -233,7 +274,7 @@ impl InnerForest { } if entries.is_empty() { - return; + return Ok(()); } let updated_root = self @@ -250,6 +291,7 @@ impl InnerForest { vault_entries = entries.len(), "Updated vault in forest" ); + Ok(()) } /// Updates the forest with storage map changes from a delta. diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs index 5bbd952c3c..08da20eaa9 100644 --- a/crates/store/src/inner_forest/tests.rs +++ b/crates/store/src/inner_forest/tests.rs @@ -86,7 +86,7 @@ fn test_update_account_with_empty_deltas() { AccountStorageDelta::default(), ); - forest.update_account(block_num, &delta); + forest.update_account(block_num, &delta).unwrap(); // Empty deltas should not create entries assert!(!forest.vault_roots.contains_key(&(account_id, block_num))); @@ -105,7 +105,7 @@ fn test_update_vault_with_fungible_asset() { vault_delta.add_asset(asset).unwrap(); let delta = dummy_partial_delta(account_id, vault_delta, AccountStorageDelta::default()); - forest.update_account(block_num, &delta); + forest.update_account(block_num, &delta).unwrap(); let vault_root = forest.vault_roots[&(account_id, block_num)]; assert_ne!(vault_root, EMPTY_WORD); @@ -150,7 +150,7 @@ fn test_incremental_vault_updates() { let mut vault_delta_1 = AccountVaultDelta::default(); vault_delta_1.add_asset(dummy_fungible_asset(faucet_id, 100)).unwrap(); let delta_1 = dummy_partial_delta(account_id, vault_delta_1, AccountStorageDelta::default()); - forest.update_account(block_1, &delta_1); + forest.update_account(block_1, &delta_1).unwrap(); let root_1 = forest.vault_roots[&(account_id, block_1)]; // Block 2: 150 tokens (update) @@ -158,7 +158,7 @@ fn test_incremental_vault_updates() { let mut vault_delta_2 = AccountVaultDelta::default(); vault_delta_2.add_asset(dummy_fungible_asset(faucet_id, 150)).unwrap(); let delta_2 = dummy_partial_delta(account_id, vault_delta_2, AccountStorageDelta::default()); - forest.update_account(block_2, &delta_2); + forest.update_account(block_2, &delta_2).unwrap(); let root_2 = forest.vault_roots[&(account_id, block_2)]; assert_ne!(root_1, root_2); @@ -176,7 +176,7 @@ fn test_full_state_delta_starts_from_empty_root() { vault_delta_pre.add_asset(dummy_fungible_asset(faucet_id, 999)).unwrap(); let delta_pre = dummy_partial_delta(account_id, vault_delta_pre, AccountStorageDelta::default()); - forest.update_account(block_num, &delta_pre); + forest.update_account(block_num, &delta_pre).unwrap(); assert!(forest.vault_roots.contains_key(&(account_id, block_num))); // Now create a full-state delta at the same block @@ -186,11 +186,11 @@ fn test_full_state_delta_starts_from_empty_root() { // Create a fresh forest to compare let mut fresh_forest = InnerForest::new(); - fresh_forest.update_account(block_num, &full_delta); + fresh_forest.update_account(block_num, &full_delta).unwrap(); let fresh_root = fresh_forest.vault_roots[&(account_id, block_num)]; // Update the original forest with the full-state delta - forest.update_account(block_num, &full_delta); + forest.update_account(block_num, &full_delta).unwrap(); let updated_root = forest.vault_roots[&(account_id, block_num)]; // The full-state delta should produce the same root regardless of prior state @@ -210,7 +210,7 @@ fn test_vault_state_persists_across_blocks_without_changes() { let mut vault_delta_1 = AccountVaultDelta::default(); vault_delta_1.add_asset(dummy_fungible_asset(faucet_id, 100)).unwrap(); let delta_1 = dummy_partial_delta(account_id, vault_delta_1, AccountStorageDelta::default()); - forest.update_account(block_1, &delta_1); + forest.update_account(block_1, &delta_1).unwrap(); let root_after_block_1 = forest.vault_roots[&(account_id, block_1)]; // Blocks 2-5: No changes to this account (simulated by not calling update_account) @@ -222,7 +222,7 @@ fn test_vault_state_persists_across_blocks_without_changes() { let mut vault_delta_6 = AccountVaultDelta::default(); vault_delta_6.add_asset(dummy_fungible_asset(faucet_id, 150)).unwrap(); // 100 + 50 = 150 let delta_6 = dummy_partial_delta(account_id, vault_delta_6, AccountStorageDelta::default()); - forest.update_account(block_6, &delta_6); + forest.update_account(block_6, &delta_6).unwrap(); // The root at block 6 should be different from block 1 (we added more tokens) let root_after_block_6 = forest.vault_roots[&(account_id, block_6)]; @@ -255,7 +255,7 @@ fn test_partial_delta_applies_fungible_changes_correctly() { let mut vault_delta_1 = AccountVaultDelta::default(); vault_delta_1.add_asset(dummy_fungible_asset(faucet_id, 100)).unwrap(); let delta_1 = dummy_partial_delta(account_id, vault_delta_1, AccountStorageDelta::default()); - forest.update_account(block_1, &delta_1); + forest.update_account(block_1, &delta_1).unwrap(); let root_after_100 = forest.vault_roots[&(account_id, block_1)]; // Block 2: Add 50 more tokens (partial delta with +50) @@ -264,7 +264,7 @@ fn test_partial_delta_applies_fungible_changes_correctly() { let mut vault_delta_2 = AccountVaultDelta::default(); vault_delta_2.add_asset(dummy_fungible_asset(faucet_id, 50)).unwrap(); let delta_2 = dummy_partial_delta(account_id, vault_delta_2, AccountStorageDelta::default()); - forest.update_account(block_2, &delta_2); + forest.update_account(block_2, &delta_2).unwrap(); let root_after_150 = forest.vault_roots[&(account_id, block_2)]; // Roots should be different (100 tokens vs 150 tokens) @@ -276,7 +276,7 @@ fn test_partial_delta_applies_fungible_changes_correctly() { let mut vault_delta_3 = AccountVaultDelta::default(); vault_delta_3.remove_asset(dummy_fungible_asset(faucet_id, 30)).unwrap(); let delta_3 = dummy_partial_delta(account_id, vault_delta_3, AccountStorageDelta::default()); - forest.update_account(block_3, &delta_3); + forest.update_account(block_3, &delta_3).unwrap(); let root_after_120 = forest.vault_roots[&(account_id, block_3)]; // Root should change again @@ -286,7 +286,7 @@ fn test_partial_delta_applies_fungible_changes_correctly() { // The roots should match let mut fresh_forest = InnerForest::new(); let full_delta = dummy_full_state_delta(account_id, &[dummy_fungible_asset(faucet_id, 120)]); - fresh_forest.update_account(block_3, &full_delta); + fresh_forest.update_account(block_3, &full_delta).unwrap(); let root_full_state_120 = fresh_forest.vault_roots[&(account_id, block_3)]; assert_eq!(root_after_120, root_full_state_120); @@ -309,7 +309,7 @@ fn test_partial_delta_across_long_block_range() { let mut vault_delta_1 = AccountVaultDelta::default(); vault_delta_1.add_asset(dummy_fungible_asset(faucet_id, 1000)).unwrap(); let delta_1 = dummy_partial_delta(account_id, vault_delta_1, AccountStorageDelta::default()); - forest.update_account(block_1, &delta_1); + forest.update_account(block_1, &delta_1).unwrap(); let root_after_1000 = forest.vault_roots[&(account_id, block_1)]; // Blocks 2-100: No changes to this account (simulating long gap) @@ -321,7 +321,7 @@ fn test_partial_delta_across_long_block_range() { vault_delta_101.add_asset(dummy_fungible_asset(faucet_id, 500)).unwrap(); let delta_101 = dummy_partial_delta(account_id, vault_delta_101, AccountStorageDelta::default()); - forest.update_account(block_101, &delta_101); + forest.update_account(block_101, &delta_101).unwrap(); let root_after_1500 = forest.vault_roots[&(account_id, block_101)]; // Roots should be different (1000 tokens vs 1500 tokens) @@ -330,7 +330,7 @@ fn test_partial_delta_across_long_block_range() { // Verify the final state matches a fresh forest with 1500 tokens let mut fresh_forest = InnerForest::new(); let full_delta = dummy_full_state_delta(account_id, &[dummy_fungible_asset(faucet_id, 1500)]); - fresh_forest.update_account(block_101, &full_delta); + fresh_forest.update_account(block_101, &full_delta).unwrap(); let root_full_state_1500 = fresh_forest.vault_roots[&(account_id, block_101)]; assert_eq!(root_after_1500, root_full_state_1500); @@ -356,7 +356,7 @@ fn test_update_storage_map() { let storage_delta = AccountStorageDelta::from_raw(raw); let delta = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta); - forest.update_account(block_num, &delta); + forest.update_account(block_num, &delta).unwrap(); // Verify storage root was created assert!( @@ -391,7 +391,7 @@ fn test_storage_map_incremental_updates() { let raw_1 = BTreeMap::from_iter([(slot_name.clone(), StorageSlotDelta::Map(map_delta_1))]); let storage_delta_1 = AccountStorageDelta::from_raw(raw_1); let delta_1 = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta_1); - forest.update_account(block_1, &delta_1); + forest.update_account(block_1, &delta_1).unwrap(); let root_1 = forest.storage_map_roots[&(account_id, slot_name.clone(), block_1)]; // Block 2: Insert key2 -> value2 (key1 should persist) @@ -401,7 +401,7 @@ fn test_storage_map_incremental_updates() { let raw_2 = BTreeMap::from_iter([(slot_name.clone(), StorageSlotDelta::Map(map_delta_2))]); let storage_delta_2 = AccountStorageDelta::from_raw(raw_2); let delta_2 = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta_2); - forest.update_account(block_2, &delta_2); + forest.update_account(block_2, &delta_2).unwrap(); let root_2 = forest.storage_map_roots[&(account_id, slot_name.clone(), block_2)]; // Block 3: Update key1 -> value3 @@ -411,7 +411,7 @@ fn test_storage_map_incremental_updates() { let raw_3 = BTreeMap::from_iter([(slot_name.clone(), StorageSlotDelta::Map(map_delta_3))]); let storage_delta_3 = AccountStorageDelta::from_raw(raw_3); let delta_3 = dummy_partial_delta(account_id, AccountVaultDelta::default(), storage_delta_3); - forest.update_account(block_3, &delta_3); + forest.update_account(block_3, &delta_3).unwrap(); let root_3 = forest.storage_map_roots[&(account_id, slot_name, block_3)]; // All roots should be different diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 523c7a3f21..f45c46d2c8 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -431,7 +431,7 @@ impl State { inner.blockchain.push(block_commitment); } - self.forest.write().await.apply_block_updates(block_num, account_deltas); + self.forest.write().await.apply_block_updates(block_num, account_deltas)?; info!(%block_commitment, block_num = block_num.as_u32(), COMPONENT, "apply_block successful"); @@ -1264,7 +1264,7 @@ async fn load_smt_forest( AccountDelta::try_from(account).expect("accounts from DB should not have seeds"); // Use the unified update method (will recognize it's a full-state delta) - forest.update_account(block_num, &delta); + forest.update_account(block_num, &delta)?; tracing::debug!( target: COMPONENT, From 5a8ea18255a08b221b34ccabaf15ba3747f8eb9b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Sat, 10 Jan 2026 14:47:35 +0100 Subject: [PATCH 95/95] clippy --- crates/store/src/inner_forest/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/store/src/inner_forest/tests.rs b/crates/store/src/inner_forest/tests.rs index 08da20eaa9..fb6ceb917a 100644 --- a/crates/store/src/inner_forest/tests.rs +++ b/crates/store/src/inner_forest/tests.rs @@ -124,12 +124,12 @@ fn test_compare_partial_vs_full_state_delta_vault() { vault_delta.add_asset(asset).unwrap(); let partial_delta = dummy_partial_delta(account_id, vault_delta, AccountStorageDelta::default()); - forest_partial.update_account(block_num, &partial_delta); + forest_partial.update_account(block_num, &partial_delta).unwrap(); // Approach 2: Full-state delta (simulates DB reconstruction) let mut forest_full = InnerForest::new(); let full_delta = dummy_full_state_delta(account_id, &[asset]); - forest_full.update_account(block_num, &full_delta); + forest_full.update_account(block_num, &full_delta).unwrap(); // Both approaches must produce identical vault roots let root_partial = forest_partial.vault_roots.get(&(account_id, block_num)).unwrap();