feat: cache smt every 1000 blocks (#6732)

Description --- Caches the smt every 1000 blocks by default or half the prune interval. Motivation and Context --- Loading the smt can take almost 2 mins on nextnet atm. This brings down he loading to 3 secs. How Has This Been Tested? --- Manual tests on nextnet --------- Co-authored-by: Brian Pearce <[email protected]>
tari-project · Jan 9, 2025 · d3f3502 · d3f3502
1 parent f508689
commit d3f3502
Show file tree

Hide file tree

Showing 8 changed files with 204 additions and 40 deletions.
diff --git a/applications/minotari_node/src/builder.rs b/applications/minotari_node/src/builder.rs
@@ -187,6 +187,7 @@ pub async fn configure_and_initialize_node(
             let backend = create_lmdb_database(
                 app_config.base_node.lmdb_path.as_path(),
                 app_config.base_node.lmdb.clone(),
+                app_config.base_node.storage.pruning_interval,
                 rules,
             )
             .map_err(|e| ExitError::new(ExitCode::DatabaseError, e))?;

diff --git a/applications/minotari_node/src/recovery.rs b/applications/minotari_node/src/recovery.rs
@@ -81,14 +81,25 @@ pub async fn run_recovery(node_config: &BaseNodeConfig) -> Result<(), anyhow::Er
     })?;
     let (temp_db, main_db, temp_path) = match &node_config.db_type {
         DatabaseType::Lmdb => {
-            let backend = create_lmdb_database(&node_config.lmdb_path, node_config.lmdb.clone(), rules.clone())
-                .map_err(|e| {
-                    error!(target: LOG_TARGET, "Error opening db: {}", e);
-                    anyhow!("Could not open DB: {}", e)
-                })?;
+            let backend = create_lmdb_database(
+                &node_config.lmdb_path,
+                node_config.lmdb.clone(),
+                node_config.storage.pruning_interval,
+                rules.clone(),
+            )
+            .map_err(|e| {
+                error!(target: LOG_TARGET, "Error opening db: {}", e);
+                anyhow!("Could not open DB: {}", e)
+            })?;
             let temp_path = temp_dir().join("temp_recovery");
 
-            let temp = create_lmdb_database(&temp_path, node_config.lmdb.clone(), rules.clone()).map_err(|e| {
+            let temp = create_lmdb_database(
+                &temp_path,
+                node_config.lmdb.clone(),
+                node_config.storage.pruning_interval,
+                rules.clone(),
+            )
+            .map_err(|e| {
                 error!(target: LOG_TARGET, "Error opening recovery db: {}", e);
                 anyhow!("Could not open recovery DB: {}", e)
             })?;

diff --git a/base_layer/core/src/base_node/sync/block_sync/synchronizer.rs b/base_layer/core/src/base_node/sync/block_sync/synchronizer.rs
@@ -22,7 +22,7 @@
 
 use std::{
     convert::{TryFrom, TryInto},
-    sync::Arc,
+    sync::{atomic::AtomicBool, Arc},
     time::{Duration, Instant},
 };
 
@@ -365,10 +365,11 @@ impl<'a, B: BlockchainBackend + 'static> BlockSynchronizer<'a, B> {
             );
 
             let timer = Instant::now();
+            let allow_smt_change = Arc::new(AtomicBool::new(true));
             self.db
                 .write_transaction()
                 .delete_orphan(header_hash)
-                .insert_tip_block_body(block.clone(), self.db.inner().smt())
+                .insert_tip_block_body(block.clone(), self.db.inner().smt(), allow_smt_change.clone())
                 .set_best_block(
                     block.height(),
                     header_hash,

diff --git a/base_layer/core/src/chain_storage/async_db.rs b/base_layer/core/src/chain_storage/async_db.rs
@@ -22,7 +22,7 @@
 use std::{
     mem,
     ops::RangeBounds,
-    sync::{Arc, RwLock},
+    sync::{atomic::AtomicBool, Arc, RwLock},
     time::Instant,
 };
 
@@ -394,8 +394,13 @@ impl<'a, B: BlockchainBackend + 'static> AsyncDbTransaction<'a, B> {
         self
     }
 
-    pub fn insert_tip_block_body(&mut self, block: Arc<ChainBlock>, smt: Arc<RwLock<OutputSmt>>) -> &mut Self {
-        self.transaction.insert_tip_block_body(block, smt);
+    pub fn insert_tip_block_body(
+        &mut self,
+        block: Arc<ChainBlock>,
+        smt: Arc<RwLock<OutputSmt>>,
+        allow_smt_change: Arc<AtomicBool>,
+    ) -> &mut Self {
+        self.transaction.insert_tip_block_body(block, smt, allow_smt_change);
         self
     }
 

diff --git a/base_layer/core/src/chain_storage/blockchain_database.rs b/base_layer/core/src/chain_storage/blockchain_database.rs
@@ -934,6 +934,7 @@ where B: BlockchainBackend
             Ok(v) => v,
             Err(e) => {
                 // some error happend, lets rewind the smt
+                warn!(target: LOG_TARGET, "Reloading SMT into memory from stored db via new block prepare");
                 *smt = db.calculate_tip_smt()?;
                 return Err(e);
             },
@@ -962,6 +963,7 @@ where B: BlockchainBackend
             Ok(v) => v,
             Err(e) => {
                 // some error happend, lets reset the smt to its starting state
+                warn!(target: LOG_TARGET, "Reloading SMT into memory from stored db via calculate root");
                 *smt = db.calculate_tip_smt()?;
                 return Err(e);
             },
@@ -1660,8 +1662,9 @@ fn insert_best_block(
     let timestamp = block.header().timestamp().as_u64();
     let accumulated_difficulty = block.accumulated_data().total_accumulated_difficulty;
     let expected_prev_best_block = block.block().header.prev_hash;
+    let allow_smt_change = Arc::new(AtomicBool::new(true));
     txn.insert_chain_header(block.to_chain_header())
-        .insert_tip_block_body(block, smt)
+        .insert_tip_block_body(block, smt, allow_smt_change)
         .set_best_block(
             height,
             block_hash,
@@ -2080,6 +2083,7 @@ fn reorganize_chain<T: BlockchainBackend>(
                 );
                 ChainStorageError::AccessError("write lock on smt".into())
             })?;
+            warn!(target: LOG_TARGET, "Reloading SMT into memory from stored db via reorg");
             *write_smt = backend.calculate_tip_smt()?;
             return Err(e);
         }

diff --git a/base_layer/core/src/chain_storage/db_transaction.rs b/base_layer/core/src/chain_storage/db_transaction.rs
@@ -23,7 +23,7 @@
 use std::{
     fmt,
     fmt::{Display, Error, Formatter},
-    sync::{Arc, RwLock},
+    sync::{atomic::AtomicBool, Arc, RwLock},
 };
 
 use primitive_types::U256;
@@ -171,8 +171,17 @@ impl DbTransaction {
     /// Add the BlockHeader and contents of a `Block` (i.e. inputs, outputs and kernels) to the database.
     /// If the `BlockHeader` already exists, then just the contents are updated along with the relevant accumulated
     /// data.
-    pub fn insert_tip_block_body(&mut self, block: Arc<ChainBlock>, smt: Arc<RwLock<OutputSmt>>) -> &mut Self {
-        self.operations.push(WriteOperation::InsertTipBlockBody { block, smt });
+    pub fn insert_tip_block_body(
+        &mut self,
+        block: Arc<ChainBlock>,
+        smt: Arc<RwLock<OutputSmt>>,
+        allow_smt_change: Arc<AtomicBool>,
+    ) -> &mut Self {
+        self.operations.push(WriteOperation::InsertTipBlockBody {
+            block,
+            smt,
+            allow_smt_change,
+        });
         self
     }
 
@@ -292,6 +301,7 @@ pub enum WriteOperation {
     InsertTipBlockBody {
         block: Arc<ChainBlock>,
         smt: Arc<RwLock<OutputSmt>>,
+        allow_smt_change: Arc<AtomicBool>,
     },
     InsertKernel {
         header_hash: HashOutput,
@@ -368,7 +378,11 @@ impl fmt::Display for WriteOperation {
             InsertChainHeader { header } => {
                 write!(f, "InsertChainHeader(#{} {})", header.height(), header.hash())
             },
-            InsertTipBlockBody { block, smt: _ } => write!(
+            InsertTipBlockBody {
+                block,
+                smt: _,
+                allow_smt_change: _,
+            } => write!(
                 f,
                 "InsertTipBlockBody({}, {})",
                 block.accumulated_data().hash,