Merge branch 'next' into mirko/readmes

Author: Mirko-von-Leipzig

CHANGELOG.md

@@ -54,6 +54,8 @@
 - Added a `miden-ntx-builder bootstrap` command that initializes the ntx-builder database with the genesis block fetched from the node RPC. The `start` command now requires a bootstrapped database instead of fetching the genesis block from the committed-block subscription on first run ([#2149](https://github.com/0xMiden/node/pull/2149)).
 - Added `--tx-expiration-delta` (env `MIDEN_NODE_NTX_BUILDER_TX_EXPIRATION_DELTA`, default `30`) to the network transaction builder: submitted network transactions now expire on-chain after this many blocks, and the builder reuses the same delta as the local window before resubmitting a transaction that has not landed ([#2148](https://github.com/0xMiden/node/pull/2148)).
 - Added a `miden-ntx-builder bootstrap` command that initializes the ntx-builder database from a trusted genesis block file. The `start` command now requires a bootstrapped database instead of fetching the genesis block from the committed-block subscription on first run ([#2149](https://github.com/0xMiden/node/pull/2149)).
+- Persisted the genesis commitment in the ntx-builder at bootstrap and sent it in the RPC `Accept` header so the node accepts its write transactions, and added RPC client implementations ([#2162](https://github.com/0xMiden/node/pull/2162)).
+- Persisted attachments of private output notes when applying a block, so they are now returned by `GetNotesById` ([#2172](https://github.com/0xMiden/node/pull/2172)).
 
 ## v0.14.11 (TBD)
 

bin/node/src/commands/rpc.rs

@@ -3,7 +3,6 @@ use std::num::{NonZeroU32, NonZeroU64};
 use std::time::Duration;
 
 use anyhow::Context;
-use miden_node_rpc::NetworkTxAuth;
 use miden_node_utils::clap::{GrpcOptionsExternal, duration_to_human_readable_string};
 use tonic::metadata::AsciiMetadataValue;
 use url::Url;
@@ -44,13 +43,12 @@ impl RpcOptions {
         }
     }
 
-    pub(super) fn network_tx_auth(&self) -> anyhow::Result<Option<NetworkTxAuth>> {
+    pub(super) fn network_tx_auth(&self) -> anyhow::Result<Option<AsciiMetadataValue>> {
         self.network_tx_auth_header_value
             .as_deref()
             .map(|value| {
                 value
                     .parse::<AsciiMetadataValue>()
-                    .map(NetworkTxAuth)
                     .context("invalid rpc.network-tx-auth-header-value")
             })
             .transpose()

bin/ntx-builder/src/actor/execute.rs

@@ -156,6 +156,9 @@ pub struct NtxContext {
 
     /// Pre-compiled transaction script that sets the network tx's on-chain expiration delta. Cloned
     /// into the [`TransactionArgs`] of the executed transaction.
+    ///
+    /// TEMP: disabled until the resolution of <https://github.com/0xMiden/protocol/issues/3027>
+    #[expect(dead_code)]
     expiration_script: TransactionScript,
 
     /// [`ExponentialBuilder`] used to back off retries on transient request failures.
@@ -382,7 +385,11 @@ impl NtxContext {
 
         // Attach the pre-compiled expiration script so the submitted tx is rejected on-chain if it
         // does not land within the configured block delta.
-        let tx_args = TransactionArgs::default().with_tx_script(self.expiration_script.clone());
+        //
+        // TEMP: disabled until the resolution of https://github.com/0xMiden/protocol/issues/3027
+        // let tx_args = TransactionArgs::default().with_tx_script(self.expiration_script.clone());
+
+        let tx_args = TransactionArgs::default();
 
         Box::pin(executor.execute_transaction(
             data_store.account.id(),

bin/ntx-builder/src/actor/mod.rs

@@ -153,6 +153,7 @@ impl AccountActorContext {
             clients: GrpcClients {
                 rpc: RpcClient::new(
                     url.clone(),
+                    miden_protocol::Word::default(),
                     Duration::from_millis(100),
                     Duration::from_secs(30),
                 ),

bin/ntx-builder/src/clients/rpc.rs

@@ -5,12 +5,26 @@ use backon::{ExponentialBuilder, Retryable};
 use futures::Stream;
 use futures::stream::TryStreamExt;
 use miden_node_proto::clients::{Builder, RpcClient as InnerRpcClient};
+use miden_node_proto::domain::account::{
+    AccountDetails, AccountResponse, AccountVaultDetails, StorageMapEntries
+};
+use miden_node_proto::errors::ConversionError;
+use miden_node_proto::generated::rpc::account_request::account_detail_request::{StorageMapDetailRequest, StorageMapDetailRequests, StorageRequest, storage_map_detail_request};
+use miden_node_proto::generated::rpc::account_request::account_detail_request::storage_map_detail_request::MapKeys;
 use miden_node_proto::generated::rpc::{BlockSubscriptionRequest, BlockSubscriptionResponse};
 use miden_node_proto::generated::{self as proto};
 use miden_node_utils::ErrorReport;
 use miden_protocol::Word;
-use miden_protocol::account::{AccountId, StorageMapKey, StorageMapWitness, StorageSlotName};
-use miden_protocol::asset::{AssetVaultKey, AssetWitness};
+use miden_protocol::account::{
+    AccountCode,
+    AccountId,
+    PartialAccount,
+    PartialStorage,
+    StorageMapKey,
+    StorageMapWitness,
+    StorageSlotName,
+};
+use miden_protocol::asset::{Asset, AssetVault, AssetVaultKey, AssetWitness, PartialVault};
 use miden_protocol::block::{BlockNumber, SignedBlock};
 use miden_protocol::note::NoteScript;
 use miden_protocol::transaction::{AccountInputs, ProvenTransaction, TransactionInputs};
@@ -41,14 +55,23 @@ impl RpcClient {
     ///
     /// `backoff_initial` / `backoff_max` configure the exponential backoff schedule applied to
     /// `block_subscription` retries (the only operation that retries today).
-    pub fn new(rpc_url: Url, backoff_initial: Duration, backoff_max: Duration) -> Self {
-        Self::new_with_auth(rpc_url, None, backoff_initial, backoff_max)
+    pub fn new(
+        rpc_url: Url,
+        genesis_commitment: Word,
+        backoff_initial: Duration,
+        backoff_max: Duration,
+    ) -> Self {
+        Self::new_with_auth(rpc_url, None, genesis_commitment, backoff_initial, backoff_max)
     }
 
     /// Creates a new client with an optional metadata header for internal RPC authentication.
+    ///
+    /// `genesis_commitment` is sent as the `genesis` parameter of the `Accept` header so that the
+    /// node accepts write RPCs such as `SubmitProvenTx`, which require a matching genesis.
     pub fn new_with_auth(
         rpc_url: Url,
         rpc_auth_header_value: Option<AsciiMetadataValue>,
+        genesis_commitment: Word,
         backoff_initial: Duration,
         backoff_max: Duration,
     ) -> Self {
@@ -58,7 +81,7 @@ impl RpcClient {
             .without_tls()
             .without_timeout()
             .without_metadata_version()
-            .without_metadata_genesis();
+            .with_metadata_genesis(genesis_commitment.to_hex());
         let builder = match rpc_auth_header_value {
             Some(value) => builder.with_auth_header_value(value),
             None => builder.without_auth_header(),
@@ -151,45 +174,195 @@ fn decode_block_subscription_response(
 // ACTOR-PATH METHODS
 // ================================================================================================
 //
-// The actor module still references these methods. PR 1 keeps the actor code in tree as dead
-// code (it is not spawned), so the methods exist as stubs to preserve compilation. PR 2 wires
-// them through the appropriate RPC gRPC service.
-
-#[expect(clippy::unused_async)]
+// Required endpoint implementations for the NTX `DataStore` implementation
 impl RpcClient {
+    /// Fetches the transaction inputs for a specific account.
+    ///
+    /// These inputs reference a specific `block_num`, and include a minimal partial account,
+    /// plus its witness.
     pub async fn get_account_inputs(
         &self,
-        _account_id: AccountId,
-        _block_num: BlockNumber,
+        account_id: AccountId,
+        block_num: BlockNumber,
     ) -> Result<AccountInputs, RpcError> {
-        unimplemented!("get_account_inputs is rewired in PR 2 of the ntx-builder refactor")
+        // Only request account code
+        let request = proto::rpc::AccountRequest {
+            account_id: Some(proto::account::AccountId { id: account_id.to_bytes() }),
+            block_num: Some(block_num.into()),
+            // TODO: should these commitments be cached on the NTX builder?
+            details: Some(proto::rpc::account_request::AccountDetailRequest {
+                code_commitment: Some(Word::default().into()),
+                asset_vault_commitment: None, //
+                storage_request: None,
+            }),
+        };
+
+        let response = self.get_account(request).await?;
+        let details = response.details.as_ref().ok_or_else(|| {
+            RpcError::InvalidResponse("response did not include account details".into())
+        })?;
+        let partial_account = build_minimal_partial_account(details)?;
+
+        Ok(AccountInputs::new(partial_account, response.witness))
     }
 
+    /// Fetches asset vault witnesses for the given keys at the reference block.
     pub async fn get_vault_asset_witnesses(
         &self,
-        _account_id: AccountId,
-        _vault_keys: BTreeSet<AssetVaultKey>,
-        _block_num: Option<BlockNumber>,
+        account_id: AccountId,
+        vault_keys: BTreeSet<AssetVaultKey>,
+        block_num: Option<BlockNumber>,
     ) -> Result<Vec<AssetWitness>, RpcError> {
-        unimplemented!("get_vault_asset_witnesses is rewired in PR 2 of the ntx-builder refactor")
+        if vault_keys.is_empty() {
+            return Ok(Vec::new());
+        }
+
+        let request = proto::rpc::AccountRequest {
+            account_id: Some(proto::account::AccountId { id: account_id.to_bytes() }),
+            block_num: block_num.map(Into::into),
+            details: Some(proto::rpc::account_request::AccountDetailRequest {
+                code_commitment: None,
+                asset_vault_commitment: Some(Word::default().into()),
+                storage_request: None,
+            }),
+        };
+
+        let response = self.get_account(request).await?;
+        let assets: Vec<Asset> = match response.details.map(|details| details.vault_details) {
+            Some(AccountVaultDetails::Assets(assets)) => assets,
+            Some(AccountVaultDetails::LimitExceeded) => {
+                // NOTE: in the tx kernel, `get_vault_asset_witnesses` is called either for single
+                // asset keys, or when pre-loading all the assets related to input notes involved in
+                // the transaction. This should never exceed the maximum amount of keys you can
+                // request to RPC, but this needs double-checking. If it able to exceed them,
+                // batching needs to be implemented as a workaround.
+                panic!("should never exceed maximum number of requested keys")
+            },
+            None => Vec::new(),
+        };
+
+        let vault =
+            AssetVault::new(&assets).map_err(|err| RpcError::InvalidResponse(err.as_report()))?;
+
+        Ok(vault_keys.into_iter().map(|key| vault.open(key)).collect())
     }
 
+    /// Fetches a storage map witness for a single key at the reference block.
     pub async fn get_storage_map_witness(
         &self,
-        _account_id: AccountId,
-        _slot_name: StorageSlotName,
-        _map_key: StorageMapKey,
-        _block_num: Option<BlockNumber>,
+        account_id: AccountId,
+        slot_name: StorageSlotName,
+        map_key: StorageMapKey,
+        block_num: Option<BlockNumber>,
     ) -> Result<StorageMapWitness, RpcError> {
-        unimplemented!("get_storage_map_witness is rewired in PR 2 of the ntx-builder refactor")
+        let request = proto::rpc::AccountRequest {
+            account_id: Some(proto::account::AccountId { id: account_id.to_bytes() }),
+            block_num: block_num.map(Into::into),
+            details: Some(proto::rpc::account_request::AccountDetailRequest {
+                code_commitment: None,
+                asset_vault_commitment: None,
+                storage_request: Some(StorageRequest::StorageMaps(StorageMapDetailRequests {
+                    storage_maps: vec![StorageMapDetailRequest {
+                        slot_name: slot_name.to_string(),
+                        slot_data: Some(storage_map_detail_request::SlotData::MapKeys(MapKeys {
+                            map_keys: vec![map_key.into()],
+                        })),
+                    }],
+                })),
+            }),
+        };
+
+        let response = self.get_account(request).await?;
+        let details = response.details.as_ref().ok_or_else(|| {
+            RpcError::InvalidResponse("response did not include account details".into())
+        })?;
+
+        let map_details = details
+            .storage_details
+            .map_details
+            .iter()
+            .find(|detail| detail.slot_name == slot_name)
+            .ok_or_else(|| {
+                RpcError::InvalidResponse(format!(
+                    "response is missing storage map details for slot {slot_name}"
+                ))
+            })?;
+
+        let StorageMapEntries::EntriesWithProofs(proofs) = &map_details.entries else {
+            return Err(RpcError::InvalidResponse(
+                "response did not include storage map entry proofs".into(),
+            ));
+        };
+
+        let proof = proofs.first().cloned().ok_or_else(|| {
+            RpcError::InvalidResponse(
+                "response did not include a proof for the requested key".into(),
+            )
+        })?;
+
+        StorageMapWitness::new(proof, [map_key])
+            .map_err(|err| RpcError::InvalidResponse(err.as_report()))
     }
 
+    /// Fetches a note script by its root, returning `None` if the node does not know it.
+    #[instrument(target = COMPONENT, name = "ntx.rpc.client.get_note_script_by_root", skip_all, err)]
     pub async fn get_note_script_by_root(
         &self,
-        _script_root: Word,
+        script_root: Word,
     ) -> Result<Option<NoteScript>, RpcError> {
-        unimplemented!("get_note_script_by_root is rewired in PR 2 of the ntx-builder refactor")
+        let request = proto::note::NoteScriptRoot { root: Some(script_root.into()) };
+
+        let script = self
+            .inner
+            .clone()
+            .get_note_script_by_root(request)
+            .await
+            .map_err(RpcError::GrpcClientError)?
+            .into_inner()
+            .script;
+
+        script.map(NoteScript::try_from).transpose().map_err(RpcError::Conversion)
     }
+
+    /// Issues a `GetAccount` request and decodes the response into the domain [`AccountResponse`].
+    async fn get_account(
+        &self,
+        request: proto::rpc::AccountRequest,
+    ) -> Result<AccountResponse, RpcError> {
+        let response = self
+            .inner
+            .clone()
+            .get_account(request)
+            .await
+            .map_err(RpcError::GrpcClientError)?
+            .into_inner();
+
+        AccountResponse::try_from(response).map_err(RpcError::Conversion)
+    }
+}
+
+/// Builds a minimal partial account from account details.
+fn build_minimal_partial_account(details: &AccountDetails) -> Result<PartialAccount, RpcError> {
+    let code_bytes = details
+        .account_code
+        .as_ref()
+        .ok_or_else(|| RpcError::InvalidResponse("response did not include account code".into()))?;
+    let account_code = AccountCode::read_from_bytes(code_bytes).map_err(RpcError::Deserialize)?;
+
+    let partial_storage = PartialStorage::new(details.storage_details.header.clone(), [])
+        .map_err(|err| RpcError::InvalidResponse(err.as_report()))?;
+
+    let partial_vault = PartialVault::new(details.account_header.vault_root());
+
+    PartialAccount::new(
+        details.account_header.id(),
+        details.account_header.nonce(),
+        account_code,
+        partial_storage,
+        partial_vault,
+        None,
+    )
+    .map_err(|err| RpcError::InvalidResponse(err.as_report()))
 }
 
 // RPC ERROR
@@ -201,4 +374,8 @@ pub enum RpcError {
     GrpcClientError(#[source] tonic::Status),
     #[error("failed to deserialize RPC payload")]
     Deserialize(#[source] miden_protocol::utils::serde::DeserializationError),
+    #[error("failed to convert RPC response")]
+    Conversion(#[source] ConversionError),
+    #[error("invalid RPC response: {0}")]
+    InvalidResponse(String),
 }

bin/ntx-builder/src/db/migrations.rs

@@ -30,7 +30,7 @@ mod tests {
     use super::*;
 
     const EXPECTED_SCHEMA_HASHES: [SchemaHash; 1] = [SchemaHash::from_hex(
-        "e7383731af6f594a2f84ea8c3863325f0219899cff13e1396630c4ea8fed8157",
+        "c631b773787903a3dd5ea4df5e7374119b3f02b35bacf14d11eacd8d8500e3d9",
     )];
 
     #[test]

bin/ntx-builder/src/db/migrations/001_initial.sql

@@ -11,6 +11,9 @@ CREATE TABLE chain_state (
     block_header    BLOB    NOT NULL,
     -- Serialized PartialMmr corresponding to `block_header`.
     chain_mmr       BLOB    NOT NULL,
+    -- Serialized genesis block commitment (Word). Set once at bootstrap and retained across tip
+    -- updates; used for the `genesis` Accept-header param required by write RPCs.
+    genesis_commitment BLOB NOT NULL,
 
     CONSTRAINT chain_state_block_num_is_u32 CHECK (block_num BETWEEN 0 AND 0xFFFFFFFF)
 );