fix: templates (#6731)

Description
---
If the mempool is out of sync, this will give the mempool 10*10ms time
to process the latest block to ensure that it is up to sync with the
latest blocks.

Motivation and Context
---
Recent changes to templates can cause miners to get invalid templates.
We need to ensure that mempool has processed the latest tip height
before asking for a new template as the new template might not be valid

base_layer/core/src/base_node/comms_interface/inbound_handlers.rs

@@ -62,6 +62,7 @@ const LOG_TARGET: &str = "c::bn::comms_interface::inbound_handler";
 const MAX_REQUEST_BY_BLOCK_HASHES: usize = 100;
 const MAX_REQUEST_BY_KERNEL_EXCESS_SIGS: usize = 100;
 const MAX_REQUEST_BY_UTXO_HASHES: usize = 100;
+const MAX_MEMPOOL_TIMEOUT: u64 = 150;
 
 /// Events that can be published on the Validated Block Event Stream
 /// Broadcast is to notify subscribers if this is a valid propagated block event
@@ -257,16 +258,28 @@ where B: BlockchainBackend + 'static
             },
             NodeCommsRequest::GetNewBlockTemplate(request) => {
                 let best_block_header = self.blockchain_db.fetch_tip_header().await?;
-                let last_seen_hash = self.mempool.get_last_seen_hash().await?;
-                let mut is_mempool_synced = true;
-                if last_seen_hash != FixedHash::default() && best_block_header.hash() != &last_seen_hash {
+                let mut last_seen_hash = self.mempool.get_last_seen_hash().await?;
+                let mut is_mempool_synced = false;
+                let start = Instant::now();
+                // this will wait a max of 150ms by default before returning anyway with a potential broken template
+                // We need to ensure the mempool has seen the latest base node height before we can be confident the
+                // template is correct
+                while !is_mempool_synced && start.elapsed().as_millis() < MAX_MEMPOOL_TIMEOUT.into() {
+                    if best_block_header.hash() == &last_seen_hash {
+                        is_mempool_synced = true;
+                    } else {
+                        tokio::time::sleep(std::time::Duration::from_millis(10)).await;
+                        last_seen_hash = self.mempool.get_last_seen_hash().await?;
+                    }
+                }
+
+                if !is_mempool_synced {
                     warn!(
                         target: LOG_TARGET,
                         "Mempool out of sync - last seen hash '{}' does not match the tip hash '{}'. This condition \
                          should auto correct with the next block template request",
                         last_seen_hash, best_block_header.hash()
                     );
-                    is_mempool_synced = false;
                 }
                 let mut header = BlockHeader::from_previous(best_block_header.header());
                 let constants = self.consensus_manager.consensus_constants(header.height);