feat: Use streaming merkle trees for block proof hashes

hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto

@@ -38,7 +38,6 @@ import "block/stream/output/transaction_output.proto";
 import "block/stream/output/transaction_result.proto";
 import "block/stream/trace/trace_data.proto";
 import "block/stream/output/block_footer.proto";
-import "services/state/blockstream/streaming_tree_snapshot.proto";
 
 /**
  * A single item within a block stream.
@@ -390,3 +389,26 @@ message RedactedItem {
      */
     SubMerkleTree tree = 3;
 }
+
+/**
+ * Identifier for each sub-tree of the block root fixed size tree
+ */
+enum SubMerkleTree {
+    ITEM_TYPE_UNSPECIFIED = 0;  // Default value, required best practice
+    PREVIOUS_BLOCK_ROOT = 1;
+    PREVIOUS_ROOTS_TREE = 2;
+    PREVIOUS_BLOCK_START_STATE = 3;
+    CONSENSUS_HEADER_ITEMS = 4;
+    INPUT_ITEMS_TREE = 5;
+    OUTPUT_ITEMS_TREE = 6;
+    STATE_CHANGE_ITEMS_TREE = 7;
+    TRACE_DATA_ITEMS_TREE = 8;
+    FUTURE_1 = 9; // these place holders for future use sub trees, will be renamed if they are used later
+    FUTURE_2 = 10;
+    FUTURE_3 = 11;
+    FUTURE_4 = 12;
+    FUTURE_5 = 13;
+    FUTURE_6 = 14;
+    FUTURE_7 = 15;
+    FUTURE_8 = 16;
+}

hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto

@@ -152,45 +152,32 @@ message BlockStreamInfo {
 
     /**
      * A SHA2-384 hash value.<br/>
-     * This is the hash of the "consensus headers" subtree for this block.
+     * This is the final hash of the "consensus headers" subtree for this block.
      */
-    bytes consensus_header_tree_root_hash = 14 [deprecated = true];
+    bytes consensus_header_root_hash = 14;
 
     /**
      * A SHA2-384 hash value.<br/>
-     * This is the hash of the "trace data" subtree for this block.
+       * This is the final hash of the "output" subtree for this block.
      */
-    bytes trace_data_tree_root_hash = 15 [deprecated = true];
-
-    /**
-     * A SHA2-384 hash value.<br/>
-     * This is the hash of the "output" subtree for this block.
-     */
-    bytes output_tree_root_hash = 16 [deprecated = true];
-
-    /**
-     * A SHA2-384 hash value.<br/>
-     * This is the final hash of the "consensus" subtree for this block.
-     */
-    bytes consensus_header_root_hash = 17;
-
-    /**
-     * A SHA2-384 hash value.<br/>
-     * This is the final hash of the "output" subtree for this block.
-     */
-    bytes output_item_root_hash = 18;
+    bytes output_item_root_hash = 15;
 
     /**
      * A SHA2-384 hash value.<br/>
      * This is the final hash of the "trace data" subtree for this block.
      */
-    bytes trace_data_root_hash = 19;
+    bytes trace_data_root_hash = 16;
 
     /**
      * The intermediate hashes needed for subroot 2 in the block merkle
      * tree structure. These hashes SHALL include the minimum required
      * block root hashes needed to construct subroot 2's final state at
      * the end of the previous block.
      */
-    repeated bytes intermediate_previous_block_root_hashes = 20;
+    repeated bytes intermediate_previous_block_root_hashes = 17;
+
+    /**
+     * The number of leaves in the intermediate block roots subtree.
+     */
+    uint64 intermediate_block_roots_leaf_count = 18;
 }

hedera-node/hapi-utils/src/main/java/com/hedera/node/app/hapi/utils/CommonUtils.java

@@ -6,6 +6,7 @@
 import static com.hedera.node.app.hapi.utils.CommonPbjConverters.toPbj;
 import static java.lang.System.arraycopy;
 import static java.util.Objects.requireNonNull;
+import static org.hiero.base.crypto.Cryptography.NULL_HASH;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.primitives.Longs;
@@ -22,6 +23,7 @@
 import com.hederahashgraph.api.proto.java.TransactionBody;
 import com.hederahashgraph.api.proto.java.TransactionOrBuilder;
 import edu.umd.cs.findbugs.annotations.NonNull;
+import edu.umd.cs.findbugs.annotations.Nullable;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.time.Instant;
@@ -185,4 +187,13 @@ public static long clampedAdd(final long addendA, final long addendB) {
             return addendA > 0 ? Long.MAX_VALUE : Long.MIN_VALUE;
         }
     }
+
+    /**
+     * Returns the given hash if it is non-null and non-empty; otherwise, returns {@code NULL_HASH}
+     * @param maybeHash the possibly null or empty hash
+     * @return the given hash or {@code NULL_HASH} if the given hash is null or empty
+     */
+    public static Bytes inputOrNullHash(@Nullable final Bytes maybeHash) {
+        return (maybeHash != null && maybeHash.length() > 0) ? maybeHash : NULL_HASH.getBytes();
+    }
 }

hedera-node/hapi-utils/src/test/java/com/hedera/node/app/hapi/utils/CommonUtilsTest.java

@@ -49,6 +49,7 @@
 import static com.hederahashgraph.api.proto.java.HederaFunctionality.UncheckedSubmit;
 import static com.hederahashgraph.api.proto.java.HederaFunctionality.UtilPrng;
 import static com.hederahashgraph.api.proto.java.ResponseType.ANSWER_ONLY;
+import static org.hiero.base.crypto.Cryptography.NULL_HASH;
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -59,6 +60,7 @@
 import com.google.protobuf.ByteString;
 import com.google.protobuf.GeneratedMessage;
 import com.google.protobuf.InvalidProtocolBufferException;
+import com.hedera.pbj.runtime.io.buffer.Bytes;
 import com.hederahashgraph.api.proto.java.AccountID;
 import com.hederahashgraph.api.proto.java.ConsensusCreateTopicTransactionBody;
 import com.hederahashgraph.api.proto.java.ConsensusDeleteTopicTransactionBody;
@@ -311,4 +313,23 @@ void getExpectEvmAddress() {
         final var evmAddress = asEvmAddress(123L);
         assertArrayEquals(address, evmAddress);
     }
+
+    @Test
+    void inputOrNullHashReturnsHash() {
+        final Bytes input = Bytes.wrap(new byte[] {1, 2, 3, 4, 5});
+        final var result = CommonUtils.inputOrNullHash(input);
+        assertEquals(input, result);
+    }
+
+    @Test
+    void inputOrNullHashReturnsNullHash() {
+        final var result = CommonUtils.inputOrNullHash(null);
+        assertEquals(NULL_HASH.getBytes(), result);
+    }
+
+    @Test
+    void inputOrNullHashReturnsNullHashForEmpty() {
+        final var result = CommonUtils.inputOrNullHash(Bytes.EMPTY);
+        assertEquals(NULL_HASH.getBytes(), result);
+    }
 }

hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java

@@ -1,20 +1,13 @@
 // SPDX-License-Identifier: Apache-2.0
 package com.hedera.node.app;
 
-import static com.hedera.hapi.block.stream.output.StateIdentifier.STATE_ID_BLOCK_STREAM_INFO;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.DUPLICATE_TRANSACTION;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.NOT_SUPPORTED;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.PLATFORM_NOT_ACTIVE;
 import static com.hedera.hapi.node.base.ResponseCodeEnum.UNKNOWN;
 import static com.hedera.hapi.util.HapiUtils.SEMANTIC_VERSION_COMPARATOR;
 import static com.hedera.hapi.util.HapiUtils.functionOf;
 import static com.hedera.node.app.blocks.BlockStreamManager.ZERO_BLOCK_HASH;
-import static com.hedera.node.app.blocks.impl.BlockImplUtils.combine;
-import static com.hedera.node.app.blocks.impl.BlockStreamManagerImpl.NULL_HASH;
-import static com.hedera.node.app.blocks.impl.ConcurrentStreamingTreeHasher.rootHashFrom;
-import static com.hedera.node.app.blocks.schemas.V0560BlockStreamSchema.BLOCK_STREAM_INFO_STATE_ID;
-import static com.hedera.node.app.hapi.utils.CommonUtils.noThrowSha384HashOf;
-import static com.hedera.node.app.records.impl.BlockRecordInfoUtils.blockHashByBlockNumber;
 import static com.hedera.node.app.records.schemas.V0490BlockRecordSchema.BLOCKS_STATE_ID;
 import static com.hedera.node.app.spi.workflows.record.StreamBuilder.nodeSignedTxWith;
 import static com.hedera.node.app.util.HederaAsciiArt.HEDERA;
@@ -30,9 +23,6 @@
 import static org.hiero.consensus.model.status.PlatformStatus.ACTIVE;
 import static org.hiero.consensus.model.status.PlatformStatus.STARTING_UP;
 
-import com.hedera.hapi.block.stream.BlockItem;
-import com.hedera.hapi.block.stream.output.SingletonUpdateChange;
-import com.hedera.hapi.block.stream.output.StateChange;
 import com.hedera.hapi.block.stream.output.StateChanges;
 import com.hedera.hapi.node.base.Duration;
 import com.hedera.hapi.node.base.HederaFunctionality;
@@ -41,7 +31,6 @@
 import com.hedera.hapi.node.base.Timestamp;
 import com.hedera.hapi.node.base.TransactionID;
 import com.hedera.hapi.node.state.blockrecords.BlockInfo;
-import com.hedera.hapi.node.state.blockstream.BlockStreamInfo;
 import com.hedera.hapi.node.state.roster.Roster;
 import com.hedera.hapi.node.transaction.SignedTransaction;
 import com.hedera.hapi.node.transaction.ThrottleDefinitions;
@@ -54,7 +43,6 @@
 import com.hedera.node.app.blocks.BlockStreamManager;
 import com.hedera.node.app.blocks.BlockStreamService;
 import com.hedera.node.app.blocks.InitialStateHash;
-import com.hedera.node.app.blocks.StreamingTreeHasher;
 import com.hedera.node.app.blocks.impl.BlockStreamManagerImpl;
 import com.hedera.node.app.blocks.impl.BoundaryStateChangeListener;
 import com.hedera.node.app.blocks.impl.ImmediateStateChangeListener;
@@ -1250,84 +1238,14 @@ private void initializeDagger(@NonNull final State state, @NonNull final InitTri
         notifications.register(AsyncFatalIssListener.class, daggerApp.fatalIssListener());
         if (blockStreamEnabled) {
             notifications.register(StateHashedListener.class, daggerApp.blockStreamManager());
-            daggerApp
-                    .blockStreamManager()
-                    .initLastBlockHash(
-                            switch (trigger) {
-                                case GENESIS -> ZERO_BLOCK_HASH;
-                                default ->
-                                    blockStreamService
-                                            .migratedLastBlockHash()
-                                            .orElseGet(() -> startBlockHashFrom(state));
-                            });
+            final var lastBlockHash = (trigger == GENESIS)
+                    ? ZERO_BLOCK_HASH
+                    : blockStreamService.migratedLastBlockHash().orElse(null);
+            daggerApp.blockStreamManager().init(state, lastBlockHash);
             migrationStateChanges = null;
         }
     }
 
-    /**
-     * Given the {@link BlockStreamInfo} context from a {@link State}, infers the block hash of the
-     * last block that was incorporated in this state.
-     *
-     * @param state the state to use
-     * @return the inferred block hash
-     */
-    private Bytes startBlockHashFrom(@NonNull final State state) {
-        final var blockStreamInfo = state.getReadableStates(BlockStreamService.NAME)
-                .<BlockStreamInfo>getSingleton(BLOCK_STREAM_INFO_STATE_ID)
-                .get();
-        requireNonNull(blockStreamInfo);
-        // Three of the four ingredients in the block hash are directly in the BlockStreamInfo; that is,
-        // the previous block hash, the input tree root hash, and the start of block state hash
-        final var prevBlockHash = blockStreamInfo.blockNumber() == 0L
-                ? ZERO_BLOCK_HASH
-                : blockHashByBlockNumber(
-                        blockStreamInfo.trailingBlockHashes(),
-                        blockStreamInfo.blockNumber() - 1,
-                        blockStreamInfo.blockNumber() - 1);
-        requireNonNull(prevBlockHash);
-
-        // The fourth ingredient, the state changes tree root hash, is not directly in the BlockStreamInfo, but
-        // we can recompute it based on the tree hash information and the fact the last state changes item in
-        // the block was devoted to putting the BlockStreamInfo itself into the state
-        final var stateChangesHash = stateChangesTreeRootHashFrom(blockStreamInfo);
-
-        final var level1A = combine(prevBlockHash, blockStreamInfo.startOfBlockStateHash());
-        final var level1B = combine(blockStreamInfo.consensusHeaderTreeRootHash(), blockStreamInfo.inputTreeRootHash());
-        final var level1C = combine(blockStreamInfo.outputTreeRootHash(), stateChangesHash);
-        final var level1D = combine(blockStreamInfo.traceDataTreeRootHash(), NULL_HASH);
-        final var leftParent = combine(level1A, level1B);
-        final var rightParent = combine(level1C, level1D);
-        return combine(leftParent, rightParent);
-    }
-
-    /**
-     * Given a {@link BlockStreamInfo} context, computes the state changes tree root hash that must have been
-     * computed at the end of the block that the context describes, assuming the final state change block item
-     * was the state change that put the context into the state.
-     *
-     * @param info the context to use
-     * @return the inferred output tree root hash
-     */
-    private @NonNull Bytes stateChangesTreeRootHashFrom(@NonNull final BlockStreamInfo info) {
-        // This was the last state change in the block
-        final var blockStreamInfoChange = StateChange.newBuilder()
-                .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal())
-                .singletonUpdate(SingletonUpdateChange.newBuilder()
-                        .blockStreamInfoValue(info)
-                        .build())
-                .build();
-        // And this was the last output block item
-        final var lastStateChanges = BlockItem.newBuilder()
-                .stateChanges(new StateChanges(info.blockEndTime(), List.of(blockStreamInfoChange)))
-                .build();
-        // So we can combine this last leaf's has with the size and rightmost hashes
-        // store from the pending state changes tree to recompute its final root hash
-        final var penultimateStateChangesTreeStatus = new StreamingTreeHasher.Status(
-                info.numPrecedingStateChangesItems(), info.rightmostPrecedingStateChangesTreeHashes());
-        final var lastLeafHash = noThrowSha384HashOf(BlockItem.PROTOBUF.toBytes(lastStateChanges));
-        return rootHashFrom(penultimateStateChangesTreeStatus, lastLeafHash);
-    }
-
     private void logConfiguration() {
         if (logger.isInfoEnabled()) {
             final var config = configProvider.getConfiguration();

hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java

@@ -8,6 +8,7 @@
 import com.swirlds.platform.system.state.notifications.StateHashedListener;
 import com.swirlds.state.State;
 import edu.umd.cs.findbugs.annotations.NonNull;
+import edu.umd.cs.findbugs.annotations.Nullable;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Optional;
@@ -76,13 +77,17 @@ interface Lifecycle {
     boolean hasLedgerId();
 
     /**
-     * Initializes the block stream manager after a restart or during reconnect with the hash of the last block
-     * incorporated in the state used in the restart or reconnect. (At genesis, this hash should be the
-     * {@link #ZERO_BLOCK_HASH}.)
+     * Initializes the block stream manager after a restart or during reconnect with the hashes necessary to
+     * infer the starting block tree states and the last block hash used in the restart or reconnect. At
+     * genesis, the last block hash should be the {@link #ZERO_BLOCK_HASH}. For migration scenarios, the last
+     * block hash should be the migrated block hash from {@link BlockStreamService#migratedLastBlockHash()}.
+     * In all other cases, this value should be null, and the method should calculate it from the intermediate
+     * subtree states.
      *
-     * @param blockHash the hash of the last block
+     * @param state the state to use
+     * @param lastBlockHash the hash of the last block
      */
-    void initLastBlockHash(@NonNull Bytes blockHash);
+    void init(@NonNull State state, @Nullable Bytes lastBlockHash);
 
     /**
      * Updates the internal state of the block stream manager to reflect the start of a new round.