feat: Use streaming merkle trees for block proof hashes

hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto

@@ -38,7 +38,6 @@ import "block/stream/output/transaction_output.proto";
 import "block/stream/output/transaction_result.proto";
 import "block/stream/trace/trace_data.proto";
 import "block/stream/output/block_footer.proto";
-import "services/state/blockstream/streaming_tree_snapshot.proto";
 
 /**
  * A single item within a block stream.
@@ -134,15 +133,15 @@ import "services/state/blockstream/streaming_tree_snapshot.proto";
  *    - `trace_data`
  * - Any subtree (depending on what was filtered).
  *   This item contains it's path in the tree and must be fully parsed.
- *    - `filtered_item_hash`
+ *    - `filtered_single_item`
  * - No subtree (and not part of the "proof" merkle tree)
  *    - `block_footer`
  *    - `block_proof`
  *    - `record_file`
  */
 message BlockItem {
     // Reserved for future items that require separate handling for block hash purposes.
-    reserved 14,15,16,17,18,19;
+    reserved 13,14,15,16,17,18,19,20;
 
     oneof item {
         /**
@@ -208,19 +207,9 @@ message BlockItem {
         com.hedera.hapi.block.stream.output.StateChanges state_changes = 7;
 
         /**
-         * Verification data for an item filtered from the stream.<br/>
-         * This is a hash for a merkle tree node where the contents of that
-         * part of the merkle tree have been removed from this stream.
-         * <p>
-         * Items of this type SHALL NOT be present in the full (unfiltered)
-         * block stream.<br/>
-         * Items of this type SHALL replace any item removed from a partial
-         * (filtered) block stream.<br/>
-         * Presence of `filtered_item` entries SHALL NOT prevent verification
-         * of a block, but MAY preclude verification or reconstruction of
-         * consensus state.<br/>
+         * A block item intentionally filtered from the stream.
          */
-        FilteredItemHash filtered_item_hash = 8 [deprecated = true];
+        FilteredSingleItem filtered_single_item = 8;
 
         /**
          * A signed block proof.<br/>
@@ -271,11 +260,6 @@ message BlockItem {
          * One or more Block Proof items SHALL follow this item.
          */
         com.hedera.hapi.block.stream.output.BlockFooter block_footer = 12;
-
-        /**
-         * A transaction intentionally filtered from the stream.
-         */
-        FilteredSingleItem filtered_single_item = 13;
   }
 }
 
@@ -390,3 +374,26 @@ message RedactedItem {
      */
     SubMerkleTree tree = 3;
 }
+
+/**
+ * Identifier for each sub-tree of the block root fixed size tree
+ */
+enum SubMerkleTree {
+    ITEM_TYPE_UNSPECIFIED = 0;  // Default value, required best practice
+    PREVIOUS_BLOCK_ROOT = 1;
+    PREVIOUS_ROOTS_TREE = 2;
+    PREVIOUS_BLOCK_START_STATE = 3;
+    CONSENSUS_HEADER_ITEMS = 4;
+    INPUT_ITEMS_TREE = 5;
+    OUTPUT_ITEMS_TREE = 6;
+    STATE_CHANGE_ITEMS_TREE = 7;
+    TRACE_DATA_ITEMS_TREE = 8;
+    FUTURE_1 = 9; // these place holders for future use sub trees, will be renamed if they are used later
+    FUTURE_2 = 10;
+    FUTURE_3 = 11;
+    FUTURE_4 = 12;
+    FUTURE_5 = 13;
+    FUTURE_6 = 14;
+    FUTURE_7 = 15;
+    FUTURE_8 = 16;
+}

hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto

@@ -152,45 +152,32 @@ message BlockStreamInfo {
 
     /**
      * A SHA2-384 hash value.<br/>
-     * This is the hash of the "consensus headers" subtree for this block.
+     * This is the final hash of the "consensus headers" subtree for this block.
      */
-    bytes consensus_header_tree_root_hash = 14 [deprecated = true];
+    bytes consensus_header_root_hash = 14;
 
     /**
      * A SHA2-384 hash value.<br/>
-     * This is the hash of the "trace data" subtree for this block.
+       * This is the final hash of the "output" subtree for this block.
      */
-    bytes trace_data_tree_root_hash = 15 [deprecated = true];
-
-    /**
-     * A SHA2-384 hash value.<br/>
-     * This is the hash of the "output" subtree for this block.
-     */
-    bytes output_tree_root_hash = 16 [deprecated = true];
-
-    /**
-     * A SHA2-384 hash value.<br/>
-     * This is the final hash of the "consensus" subtree for this block.
-     */
-    bytes consensus_header_root_hash = 17;
-
-    /**
-     * A SHA2-384 hash value.<br/>
-     * This is the final hash of the "output" subtree for this block.
-     */
-    bytes output_item_root_hash = 18;
+    bytes output_item_root_hash = 15;
 
     /**
      * A SHA2-384 hash value.<br/>
      * This is the final hash of the "trace data" subtree for this block.
      */
-    bytes trace_data_root_hash = 19;
+    bytes trace_data_root_hash = 16;
 
     /**
      * The intermediate hashes needed for subroot 2 in the block merkle
      * tree structure. These hashes SHALL include the minimum required
      * block root hashes needed to construct subroot 2's final state at
      * the end of the previous block.
      */
-    repeated bytes intermediate_previous_block_root_hashes = 20;
+    repeated bytes intermediate_previous_block_root_hashes = 17;
+
+    /**
+     * The number of leaves in the intermediate block roots subtree.
+     */
+    uint64 intermediate_block_roots_leaf_count = 18;
 }

hedera-node/hapi-utils/src/main/java/com/hedera/node/app/hapi/utils/CommonUtils.java

@@ -6,6 +6,7 @@
 import static com.hedera.node.app.hapi.utils.CommonPbjConverters.toPbj;
 import static java.lang.System.arraycopy;
 import static java.util.Objects.requireNonNull;
+import static org.hiero.base.crypto.Cryptography.NULL_HASH;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.primitives.Longs;
@@ -22,6 +23,7 @@
 import com.hederahashgraph.api.proto.java.TransactionBody;
 import com.hederahashgraph.api.proto.java.TransactionOrBuilder;
 import edu.umd.cs.findbugs.annotations.NonNull;
+import edu.umd.cs.findbugs.annotations.Nullable;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.time.Instant;
@@ -185,4 +187,13 @@ public static long clampedAdd(final long addendA, final long addendB) {
             return addendA > 0 ? Long.MAX_VALUE : Long.MIN_VALUE;
         }
     }
+
+    /**
+     * Returns the given hash if it is non-null and non-empty; otherwise, returns {@code NULL_HASH}
+     * @param maybeHash the possibly null or empty hash
+     * @return the given hash or {@code NULL_HASH} if the given hash is null or empty
+     */
+    public static Bytes inputOrNullHash(@Nullable final Bytes maybeHash) {
+        return (maybeHash != null && maybeHash.length() > 0) ? maybeHash : NULL_HASH.getBytes();
+    }
 }

hedera-node/hapi-utils/src/test/java/com/hedera/node/app/hapi/utils/CommonUtilsTest.java

@@ -49,6 +49,7 @@
 import static com.hederahashgraph.api.proto.java.HederaFunctionality.UncheckedSubmit;
 import static com.hederahashgraph.api.proto.java.HederaFunctionality.UtilPrng;
 import static com.hederahashgraph.api.proto.java.ResponseType.ANSWER_ONLY;
+import static org.hiero.base.crypto.Cryptography.NULL_HASH;
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -59,6 +60,7 @@
 import com.google.protobuf.ByteString;
 import com.google.protobuf.GeneratedMessage;
 import com.google.protobuf.InvalidProtocolBufferException;
+import com.hedera.pbj.runtime.io.buffer.Bytes;
 import com.hederahashgraph.api.proto.java.AccountID;
 import com.hederahashgraph.api.proto.java.ConsensusCreateTopicTransactionBody;
 import com.hederahashgraph.api.proto.java.ConsensusDeleteTopicTransactionBody;
@@ -311,4 +313,23 @@ void getExpectEvmAddress() {
         final var evmAddress = asEvmAddress(123L);
         assertArrayEquals(address, evmAddress);
     }
+
+    @Test
+    void inputOrNullHashReturnsHash() {
+        final Bytes input = Bytes.wrap(new byte[] {1, 2, 3, 4, 5});
+        final var result = CommonUtils.inputOrNullHash(input);
+        assertEquals(input, result);
+    }
+
+    @Test
+    void inputOrNullHashReturnsNullHash() {
+        final var result = CommonUtils.inputOrNullHash(null);
+        assertEquals(NULL_HASH.getBytes(), result);
+    }
+
+    @Test
+    void inputOrNullHashReturnsNullHashForEmpty() {
+        final var result = CommonUtils.inputOrNullHash(Bytes.EMPTY);
+        assertEquals(NULL_HASH.getBytes(), result);
+    }
 }