From e6e072592ea1973df12a4e863411bf714ccb33d2 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Fri, 26 Sep 2025 13:29:04 -0600 Subject: [PATCH 01/63] Add initial protobufs for new block merkle tree structure Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 105 ++++++++++++++++++ .../block/stream/output/block_footer.proto | 41 +++++++ .../block/stream/output/block_proof2.proto | 62 +++++++++++ .../state/blockstream/merkle_leaf.proto | 46 ++++++++ 4 files changed, 254 insertions(+) create mode 100644 hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto create mode 100644 hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto create mode 100644 hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index 696b7bb32d0c..cbc589cd93c6 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -292,3 +292,108 @@ message FilteredItemHash { */ uint64 filtered_path = 3; } + +/** Identifer for each sub-tree of the block root fixed size tree */ +enum SubMerkleTree { + ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice + CONSENSUS_HEADER = 1; + INPUT_ITEM = 2; + OUTPUT_ITEM = 3; + STATE_CHANGE_ITEM = 4; + TRACE_ITEM = 5; + FUTURE_1 = 6; // these place holders for future use sub trees, will be renamed if they are used later + FUTURE_2 = 7; + FUTURE_3 = 8; + FUTURE_4 = 9; + FUTURE_5 = 10; + FUTURE_6 = 11; + FUTURE_7 = 12; + FUTURE_8 = 13; +} + +/** + * Verification data for an item filtered from the stream. + * + * Items of this type SHALL NOT be present in the full (unfiltered) block + * stream.
+ * Items of this type SHALL replace any item removed from a partial (filtered) + * block stream.
+ * Presence of `filtered_item` entries SHALL NOT prevent verification + * of a block, but MAY preclude verification or reconstruction + * of consensus state.
+ */ +message FilteredSingleItem { + /** + * A hash of an item filtered from the stream. + *

+ * The hash algorithm used MUST match the hash algorithm specified in + * the block header for the containing block.
+ * This field is REQUIRED. + */ + bytes item_hash = 1; + + /** + * This tells you which of the block merkle sub trees to add the item hash into + *

+ * This REQUIRED field SHALL describe the type of filtered item + */ + SubMerkleTree tree = 2; +} + +/** + * Filtered Block Item representing a complete block sub merkle tree that has been + * filtered out and replaced by a hash. + */ +message FilteredMerkleSubTree { + /** + * Root hash of a sub-merkle tree + */ + bytes subtree_root_hash = 1; + + /** + * This tells you which of the block merkle sub trees the hash is the root for + */ + SubMerkleTree tree = 2; + + /** + * The number of leaves filtered by this FilteredMerkleSubTree. + */ + uint32 filtered_leaf_count = 3; +} + +/** + * Verification data for an item redacted from the stream. + * + * Presence of `redacted_item` entries SHALL NOT prevent verification + * of a block.
+ */ +message RedactedItem { + /** + * A hash of an item redacted from the stream. + *

+ * The hash algorithm used MUST match the hash algorithm specified in + * the block header for the containing block.
+ * This field is REQUIRED. + */ + bytes item_hash = 1; + + /** + * When thise redacted item is a SignedTransaction, this value is the hash of that SignedTransaction + * directly, without the BlockItem wrapper. This is needed for event reconstruction. The + * signed_transaction_hash will only be set for event transactions, synthetic transactions will have + * empty value. + *

+ * The hash algorithm used MUST match the hash algorithm specified in + * the block header for the containing block.
+ * This value SHALL NOT be provided if the original item MUST NOT be included in an + * event hash. + */ + bytes signed_transaction_hash = 2; + + /** + * This tells you which of the block merkle sub trees to add the item hash into + *

+ * This REQUIRED field SHALL describe the type of filtered item + */ + SubMerkleTree tree = 3; +} diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto new file mode 100644 index 000000000000..60550e1f482a --- /dev/null +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto @@ -0,0 +1,41 @@ +/** + * # Block Footer + * TODO + * + * ### Keywords + * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + * document are to be interpreted as described in + * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in + * [RFC8174](https://www.ietf.org/rfc/rfc8174). + */ +syntax = "proto3"; + +package com.hedera.hapi.block.stream.output; + +// SPDX-License-Identifier: Apache-2.0 +option java_package = "com.hedera.hapi.block.stream.output.protoc"; +// <<>> This comment is special code for setting PBJ Compiler java package +option java_multiple_files = true; + +/** + * A collection of hashes of sub parts of the blocks top fixed merkle tree that are needed to compute the + * blocks root hash. These are the hashes of the first 3 nodes across the bottom of the block fixed merkle + * tree in field order. + */ +message BlockFooter { + + /** The root hash of the block, for the previous block to the one this footer belongs to. + */ + bytes previous_block_root_hash = 1; + + /** + * The root hash of a merkle tree containg the root hashes of all block from block zero up to but not + * including this current block. + */ + bytes root_hash_of_all_block_hashes_tree = 2; + + /** The root hash of the state merkle tree for the version of state at the begining of the current block + */ + bytes start_of_block_state_root_hash = 3; +} diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto new file mode 100644 index 000000000000..3d6fd716e47a --- /dev/null +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto @@ -0,0 +1,62 @@ +/** + * # Block Proof (v2) + * TODO + * Going to use a new block proof definition for now, replacing pieces until the original Block Proof + * definition isn't used. Then we'll remove the original. + * + * ### Keywords + * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + * document are to be interpreted as described in + * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in + * [RFC8174](https://www.ietf.org/rfc/rfc8174). + */ +syntax = "proto3"; + +package com.hedera.hapi.block.stream.output; + +// SPDX-License-Identifier: Apache-2.0 +option java_package = "com.hedera.hapi.block.stream.output.protoc"; +// <<>> This comment is special code for setting PBJ Compiler java package +option java_multiple_files = true; + +import "block/stream/record_file_item.proto"; + +/** + * TODO + */ +message BlockFooter { + + /** + * TODO + */ + bytes previous_block_root_hash = 1; + + /** + * TODO + */ + bytes root_hash_of_all_block_hashes_tree = 2; + + /** + * TODO + */ + bytes start_of_block_state_root_hash = 3; +} + +message TssSignedBlockProof { + bytes block_signature = 4; + + // TODO: probably will be deleted? + oneof verification_reference { + uint64 scheme_id = 6; + bytes verification_key = 7; // extracted from ledger ID? + } +} + +message SignedRecordFileProof { + /** + * A collection of RSA signatures from consensus nodes.
+ * These signatures validate the hash of the record_file_contents field. + */ + repeated com.hedera.hapi.block.stream.RecordFileSignature record_file_signatures = 1; +} diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto new file mode 100644 index 000000000000..edf6223ccd3d --- /dev/null +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto @@ -0,0 +1,46 @@ +/** + * Merkle Leaf + * TODO!! + * + * ### Keywords + * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + * document are to be interpreted as described in + * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in + * [RFC8174](https://www.ietf.org/rfc/rfc8174). + */ +syntax = "proto3"; + +package com.hedera.hapi.node.state.blockstream; + +// SPDX-License-Identifier: Apache-2.0 +import "services/timestamp.proto"; +import "platform/state/virtual_map_state.proto"; +import "block/stream/block_item.proto"; + +option java_package = "com.hedera.hapi.block.stream.protoc"; +// <<>> This comment is special code for setting PBJ Compiler java package +option java_multiple_files = true; + +/** + * TODO + */ +message MerkleLeaf { + oneof content { + /** + * TODO + */ + proto.Timestamp block_consensus_timestamp = 1; + + /** + * TODO + */ + com.hedera.hapi.block.stream.BlockItem block_item = 2; + + /** + * TODO + */ + com.hedera.hapi.platform.state.StateItem state_item = 3; + } +} + From 3f3f453a368e5e9fcc2d946bf0aec4e959b12538 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Fri, 26 Sep 2025 13:29:38 -0600 Subject: [PATCH 02/63] Add algorithm for streaming merkle tree Signed-off-by: Matt Hess --- .../impl/IncrementalStreamingHasher.java | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java new file mode 100644 index 000000000000..10addc43cb03 --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.node.app.blocks.impl; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.LinkedList; +import java.util.List; + +/** + * A class that computes a Merkle tree root hash in a streaming fashion. It supports adding leaves one by one and + * computes the root hash without storing the entire tree in memory. It uses SHA-384 as the hashing algorithm and + * follows the prefixing scheme for leaves and internal nodes. + * + *

This is not thread safe, it is assumed use by single thread.

+ */ +public class IncrementalStreamingHasher { + /** Prefix byte for hash contents for leaf nodes. */ + private static final byte[] LEAF_PREFIX = new byte[] {0}; + /** Prefix byte for hash contents for internal nodes. */ + private static final byte[] INTERNAL_NODE_PREFIX = new byte[] {2}; + /** The hashing algorithm used for computing the hashes. */ + private final MessageDigest digest; + /** A list to store intermediate hashes as we build the tree. */ + private final LinkedList hashList = new LinkedList<>(); + /** The count of leaves in the tree. */ + private long leafCount = 0; + + /** Create a new StreamingHasher with an empty state. */ + public IncrementalStreamingHasher() { + try { + digest = MessageDigest.getInstance("SHA-384"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + /** + * Create a StreamingHasher with an existing intermediate hashing state. + * This allows resuming hashing from a previous state. + * + * @param intermediateHashingState the intermediate hashing state + */ + public IncrementalStreamingHasher(List intermediateHashingState) { + this(); + this.hashList.addAll(intermediateHashingState); + } + + /** + * Add a new leaf to the Merkle tree. + * + * @param data the data for the new leaf + */ + public void addLeaf(byte[] data) { + final long i = leafCount; + final byte[] e = hashLeaf(data); + hashList.add(e); + for (long n=i; (n & 1L) == 1; n >>= 1) { + final byte[] y = hashList.removeLast(); + final byte[] x = hashList.removeLast(); + hashList.add(hashInternalNode(x, y)); + } + leafCount ++; + } + + /** + * Compute the Merkle tree root hash from the current state. This does not modify the internal state, so can be + * called at any time and more leaves can be added afterward. + * + * @return the Merkle tree root hash + */ + public byte[] computeRootHash() { + byte[] merkleRootHash = hashList.getLast(); + for (int i = hashList.size() - 2; i >= 0; i--) { + merkleRootHash = hashInternalNode(hashList.get(i), merkleRootHash); + } + return merkleRootHash; + } + + /** + * Get the current intermediate hashing state. This can be used to save the state and resume hashing later. + * + * @return the intermediate hashing state + */ + public List intermediateHashingState() { + return hashList; + } + + /** + * Get the number of leaves added to the tree so far. + * + * @return the number of leaves + */ + public long leafCount() { + return leafCount; + } + + /** + * Hash a leaf node with the appropriate prefix. + * + * @param leafData the data of the leaf + * @return the hash of the leaf node + */ + private byte[] hashLeaf(final byte[] leafData) { + digest.update(LEAF_PREFIX); + return digest.digest(leafData); + } + + /** + * Hash an internal node by combining the hashes of its two children with the appropriate prefix. + * + * @param firstChild the hash of the first child + * @param secondChild the hash of the second child + * @return the hash of the internal node + */ + private byte[] hashInternalNode(final byte[] firstChild, final byte[] secondChild) { + digest.update(INTERNAL_NODE_PREFIX); + digest.update(firstChild); + return digest.digest(secondChild); + } +} From 8df40501c64e27df217b1f7763c356aad11820df Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Fri, 26 Sep 2025 13:51:36 -0600 Subject: [PATCH 03/63] Update BlockItem with new definition Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 94 +++---------------- .../block/stream/output/block_proof2.proto | 62 ------------ .../app/blocks/impl/BlockStreamBuilder.java | 16 +--- .../blocks/impl/BlockStreamManagerImpl.java | 8 +- .../app/blocks/BlockStreamBuilderTest.java | 43 +++++---- .../impl/BlockStreamManagerImplTest.java | 4 +- 6 files changed, 47 insertions(+), 180 deletions(-) delete mode 100644 hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index cbc589cd93c6..b326ca5e8c49 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -37,6 +37,7 @@ import "block/stream/output/state_changes.proto"; import "block/stream/output/transaction_output.proto"; import "block/stream/output/transaction_result.proto"; import "block/stream/trace/trace_data.proto"; +import "block/stream/output/block_footer.proto"; /** * A single item within a block stream. @@ -68,9 +69,11 @@ import "block/stream/trace/trace_data.proto"; * transaction_result * (optional) transaction_output * (optional) repeated state_changes + * (optional) filtered_single_item * } + * block_footer + * repeated block_proof * } - * state_proof * ``` * * A filtered stream may exclude some items above, depending on filter @@ -115,6 +118,7 @@ import "block/stream/trace/trace_data.proto"; * - The "BridgeTransform" field is 24 (24 modulo 10 is 4, so it is Trace Data). * * #### Initial Field assignment to subtree categories. + * TODO: REDEFINE * - Consensus Headers * - `event_header` * - `round_header` @@ -203,96 +207,22 @@ message BlockItem { com.hedera.hapi.block.stream.output.StateChanges state_changes = 7; /** - * Verification data for an item filtered from the stream.
- * This is a hash for a merkle tree node where the contents of that - * part of the merkle tree have been removed from this stream. - *

- * Items of this type SHALL NOT be present in the full (unfiltered) - * block stream.
- * Items of this type SHALL replace any item removed from a partial - * (filtered) block stream.
- * Presence of `filtered_item` entries SHALL NOT prevent verification - * of a block, but MAY preclude verification or reconstruction of - * consensus state.
+ * TODO */ - FilteredItemHash filtered_item_hash = 8; + FilteredSingleItem filtered_single_item = 8; /** - * A signed block proof.
- * The signed merkle proof for this block. This will validate - * a "virtual" merkle tree containing the previous block "virtual" - * root, an "input" subtree, an "output" subtree, and - * a "state changes" subtree. - *

- * This item is not part of the block stream hash chain/tree, and - * MUST follow after the end of a block. + * TODO */ - BlockProof block_proof = 9; + com.hedera.hapi.block.stream.output.BlockFooter block_footer = 9; /** - * A record file and associated data. - *

- * This MUST contain a single Record file, associated Sidecar files, - * and data from related Signature files. - * If this item is present, special treatment is - * REQUIRED for this block. - *

    - *
  • The block SHALL NOT have a `BlockHeader`.
    • - *
  • The block SHALL NOT have a `BlockProof`.
    • - *
  • The block SHALL contain _exactly one_ `RecordFileItem`.
    • - *
  • The block SHALL NOT contain any item other than a - * `RecordFileItem`.
    • - *
  • The content of the `RecordFileItem` MUST be validated using - * the signature data and content provided within according to - * the process used for Record Files prior to the creation - * of Block Stream.
    • - *
+ * TODO */ - RecordFileItem record_file = 10; - - /** - * A trace data. - *

- * Any informational trace data MAY be described by - * stream items of this type.
- */ - com.hedera.hapi.block.stream.trace.TraceData trace_data = 11; + BlockProof block_proof = 10; } } -/** - * Verification data for an item filtered from the stream. - * - * Items of this type SHALL NOT be present in the full (unfiltered) block - * stream.
- * Items of this type SHALL replace any item removed from a partial (filtered) - * block stream.
- * Presence of `filtered_item` entries SHALL NOT prevent verification - * of a block, but MAY preclude verification or reconstruction - * of consensus state.
- */ -message FilteredItemHash { - /** - * A hash of an item filtered from the stream. - *

- * The hash algorithm used MUST match the hash algorithm specified in - * the block header for the containing block.
- * This field is REQUIRED. - */ - bytes item_hash = 1; - - /** - * A record of the merkle path to the item that was filtered - * from the stream.
- * This path begins at the root of the block proof merkle tree. - *

- * This REQUIRED field SHALL describe the full path in the virtual - * merkle tree constructed for the block proof that contained the - * item filtered from the stream. - */ - uint64 filtered_path = 3; -} - /** Identifer for each sub-tree of the block root fixed size tree */ enum SubMerkleTree { ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice @@ -378,7 +308,7 @@ message RedactedItem { bytes item_hash = 1; /** - * When thise redacted item is a SignedTransaction, this value is the hash of that SignedTransaction + * When this redacted item is a SignedTransaction, this value is the hash of that SignedTransaction * directly, without the BlockItem wrapper. This is needed for event reconstruction. The * signed_transaction_hash will only be set for event transactions, synthetic transactions will have * empty value. diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto deleted file mode 100644 index 3d6fd716e47a..000000000000 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto +++ /dev/null @@ -1,62 +0,0 @@ -/** - * # Block Proof (v2) - * TODO - * Going to use a new block proof definition for now, replacing pieces until the original Block Proof - * definition isn't used. Then we'll remove the original. - * - * ### Keywords - * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", - * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this - * document are to be interpreted as described in - * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in - * [RFC8174](https://www.ietf.org/rfc/rfc8174). - */ -syntax = "proto3"; - -package com.hedera.hapi.block.stream.output; - -// SPDX-License-Identifier: Apache-2.0 -option java_package = "com.hedera.hapi.block.stream.output.protoc"; -// <<>> This comment is special code for setting PBJ Compiler java package -option java_multiple_files = true; - -import "block/stream/record_file_item.proto"; - -/** - * TODO - */ -message BlockFooter { - - /** - * TODO - */ - bytes previous_block_root_hash = 1; - - /** - * TODO - */ - bytes root_hash_of_all_block_hashes_tree = 2; - - /** - * TODO - */ - bytes start_of_block_state_root_hash = 3; -} - -message TssSignedBlockProof { - bytes block_signature = 4; - - // TODO: probably will be deleted? - oneof verification_reference { - uint64 scheme_id = 6; - bytes verification_key = 7; // extracted from ledger ID? - } -} - -message SignedRecordFileProof { - /** - * A collection of RSA signatures from consensus nodes.
- * These signatures validate the hash of the record_file_contents field. - */ - repeated com.hedera.hapi.block.stream.RecordFileSignature record_file_signatures = 1; -} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java index 4e67160dd6da..11ea1bd54e69 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java @@ -543,15 +543,7 @@ private T toView(@NonNull final BlockItemsTranslator translator, @NonNull fi } List logs = null; for (final var item : blockItems.subList(j, n)) { - if (item.hasTraceData()) { - final var traceData = item.traceDataOrThrow(); - if (traceData.hasEvmTraceData()) { - if (logs == null) { - logs = new ArrayList<>(); - } - logs.addAll(traceData.evmTraceDataOrThrow().logs()); - } - } + // TODO: new trace data implementation } return (T) switch (view) { @@ -677,7 +669,7 @@ public Output build(final boolean topLevel, @Nullable final List ba builder.logs(logs); } blockItems.add(BlockItem.newBuilder() - .traceData(TraceData.newBuilder().evmTraceData(builder)) + // TODO: re-add trace data .build()); } @@ -689,7 +681,7 @@ public Output build(final boolean topLevel, @Nullable final List ba .automaticTokenAssociations( automaticTokenAssociations.getLast().accountId()); blockItems.add(BlockItem.newBuilder() - .traceData(TraceData.newBuilder().autoAssociateTraceData(builder)) + // TODO: re-add trace data .build()); } // message submit trace data @@ -698,7 +690,7 @@ public Output build(final boolean topLevel, @Nullable final List ba .sequenceNumber(sequenceNumber) .runningHash(runningHash); blockItems.add(BlockItem.newBuilder() - .traceData(TraceData.newBuilder().submitMessageTraceData(builder)) + // TODO: re-add trace data .build()); } } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 3bd68150f60b..bbded212e19d 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -92,6 +92,7 @@ import org.apache.logging.log4j.Logger; import org.hiero.base.concurrent.AbstractTask; import org.hiero.base.crypto.Hash; +import org.hiero.base.exceptions.NotImplementedException; import org.hiero.consensus.model.hashgraph.Round; @Singleton @@ -754,7 +755,10 @@ protected boolean onExecute() { STATE_CHANGES, ROUND_HEADER, BLOCK_HEADER, - TRACE_DATA -> { + BLOCK_FOOTER, + BLOCK_PROOF + // Also EndBlock? + -> { MessageDigest digest = sha384DigestOrThrow(); bytes.writeTo(digest); hash = ByteBuffer.wrap(digest.digest()); @@ -793,7 +797,7 @@ protected boolean onExecute() { } case TRANSACTION_OUTPUT, BLOCK_HEADER -> outputTreeHasher.addLeaf(hash); case STATE_CHANGES -> stateChangesHasher.addLeaf(hash); - case TRACE_DATA -> traceDataHasher.addLeaf(hash); + case BLOCK_FOOTER, BLOCK_PROOF -> throw new NotImplementedException(); } final BlockHeader header = item.blockHeader(); diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java index 0c70bca01cc9..a7e291cf2523 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java @@ -141,11 +141,12 @@ void testBlockItemsWithTraceAndOutput() { assertTrue(output.hasContractCall()); final var traceItem = blockItems.get(3); - assertTrue(traceItem.hasTraceData()); - final var trace = traceItem.traceDataOrThrow(); - assertTrue(trace.hasEvmTraceData()); - final var evmTrace = trace.evmTraceDataOrThrow(); - assertEquals(usages, evmTrace.contractSlotUsages()); + // TODO: assert trace data +// assertTrue(traceItem.hasTraceData()); +// final var trace = traceItem.traceDataOrThrow(); +// assertTrue(trace.hasEvmTraceData()); +// final var evmTrace = trace.evmTraceDataOrThrow(); +// assertEquals(usages, evmTrace.contractSlotUsages()); } @Test @@ -161,14 +162,15 @@ void testBlockItemsWithAdditionalAutomaticTokenAssociationTraceData() { final var blockItems = itemsBuilder.build(false, List.of()).blockItems(); final var traceItem = blockItems.get(2); - assertThat(traceItem.hasTraceData()).isTrue(); - final var trace = traceItem.traceDataOrThrow(); - - assertThat(trace.hasAutoAssociateTraceData()).isTrue(); - final var autoAssociateTraceData = trace.autoAssociateTraceData(); - assertThat(autoAssociateTraceData).isNotNull(); - assertThat(autoAssociateTraceData.automaticTokenAssociations().accountNum()) - .isEqualTo(2); + // TODO: assert trace data +// assertThat(traceItem.hasTraceData()).isTrue(); +// final var trace = traceItem.traceDataOrThrow(); +// +// assertThat(trace.hasAutoAssociateTraceData()).isTrue(); +// final var autoAssociateTraceData = trace.autoAssociateTraceData(); +// assertThat(autoAssociateTraceData).isNotNull(); +// assertThat(autoAssociateTraceData.automaticTokenAssociations().accountNum()) +// .isEqualTo(2); } @Test @@ -179,13 +181,14 @@ void testBlockItemsWithAdditionalSubmitMsgTraceData() { final var blockItems = itemsBuilder.build(false, List.of()).blockItems(); final var traceItem = blockItems.get(2); - assertThat(traceItem.hasTraceData()).isTrue(); - final var trace = traceItem.traceDataOrThrow(); - - assertThat(trace.hasSubmitMessageTraceData()).isTrue(); - final var submitMessageTraceData = trace.submitMessageTraceData(); - assertThat(submitMessageTraceData).isNotNull(); - assertThat(submitMessageTraceData.sequenceNumber()).isEqualTo(66); + // TODO: assert trace data +// assertThat(traceItem.hasTraceData()).isTrue(); +// final var trace = traceItem.traceDataOrThrow(); +// +// assertThat(trace.hasSubmitMessageTraceData()).isTrue(); +// final var submitMessageTraceData = trace.submitMessageTraceData(); +// assertThat(submitMessageTraceData).isNotNull(); +// assertThat(submitMessageTraceData.sequenceNumber()).isEqualTo(66); } @Test diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index cbd3daa63694..531a3149130b 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -114,8 +114,8 @@ class BlockStreamManagerImplTest { private static final BlockItem FAKE_STATE_CHANGES = BlockItem.newBuilder() .stateChanges(StateChanges.newBuilder().consensusTimestamp(CONSENSUS_THEN)) .build(); - private static final BlockItem FAKE_RECORD_FILE_ITEM = - BlockItem.newBuilder().recordFile(RecordFileItem.DEFAULT).build(); + // TODO: remove, or replace with wrapped record file item + private static final BlockItem FAKE_RECORD_FILE_ITEM = null; private final InitialStateHash hashInfo = new InitialStateHash(completedFuture(ZERO_BLOCK_HASH), 0); @Mock From 5a001c31d5926a8d4a0bf55886dfebd73ad61ee1 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Fri, 3 Oct 2025 20:05:20 -0600 Subject: [PATCH 04/63] wip Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 27 +-- .../state/blockstream/block_stream_info.proto | 35 ++-- .../app/blocks/impl/BlockStreamBuilder.java | 8 +- .../blocks/impl/BlockStreamManagerImpl.java | 151 +++++++++----- .../impl/IncrementalStreamingHasher.java | 191 +++++++++--------- .../app/blocks/BlockStreamBuilderTest.java | 47 +++-- .../impl/BlockStreamManagerImplTest.java | 126 ++++++------ 7 files changed, 328 insertions(+), 257 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index b326ca5e8c49..37b6cd1423cb 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -226,19 +226,20 @@ message BlockItem { /** Identifer for each sub-tree of the block root fixed size tree */ enum SubMerkleTree { ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice - CONSENSUS_HEADER = 1; - INPUT_ITEM = 2; - OUTPUT_ITEM = 3; - STATE_CHANGE_ITEM = 4; - TRACE_ITEM = 5; - FUTURE_1 = 6; // these place holders for future use sub trees, will be renamed if they are used later - FUTURE_2 = 7; - FUTURE_3 = 8; - FUTURE_4 = 9; - FUTURE_5 = 10; - FUTURE_6 = 11; - FUTURE_7 = 12; - FUTURE_8 = 13; + PREVIOUS_ROOT_HASHES = 1; + CONSENSUS_HEADER = 2; + INPUT_ITEM = 3; + OUTPUT_ITEM = 4; + STATE_CHANGE_ITEM = 5; + TRACE_ITEM = 6; + FUTURE_1 = 7; // these place holders for future use sub trees, will be renamed if they are used later + FUTURE_2 = 8; + FUTURE_3 = 9; + FUTURE_4 = 10; + FUTURE_5 = 11; + FUTURE_6 = 12; + FUTURE_7 = 13; + FUTURE_8 = 14; } /** diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto index 2c3fffb65955..1586eaf2faac 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto @@ -102,13 +102,9 @@ message BlockStreamInfo { uint32 num_preceding_state_changes_items = 7; /** - * A concatenation of SHA2-384 hash values.
- * This is the "rightmost" values of the "output" subtree. - *

- * The subtree containing these hashes SHALL be constructed from all "output" - * `BlockItem`s in this block that _precede_ the update to this singleton. + * TODO */ - repeated bytes rightmost_preceding_state_changes_tree_hashes = 8; + repeated bytes intermediate_previous_block_root_hashes = 8; /** * A block-end consensus time stamp. @@ -150,20 +146,27 @@ message BlockStreamInfo { proto.Timestamp last_handle_time = 13; /** - * A SHA2-384 hash value.
- * This is the hash of the "consensus headers" subtree for this block. + * TODO + */ + repeated bytes intermediate_consensus_header_hashes = 14; + + /** + * TODO */ - bytes consensus_header_tree_root_hash = 14; + repeated bytes intermediate_input_block_item_hashes = 15; /** - * A SHA2-384 hash value.
- * This is the hash of the "trace data" subtree for this block. + * TODO */ - bytes trace_data_tree_root_hash = 15; + repeated bytes intermediate_output_block_item_hashes = 16; /** - * A SHA2-384 hash value.
- * This is the hash of the "output" subtree for this block. - */ - bytes output_tree_root_hash = 16; + * TODO + */ + repeated bytes intermediate_state_change_block_item_hashes = 17; + + /** + * TODO + */ + repeated bytes intermediate_trace_data_hashes = 18; } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java index 11ea1bd54e69..6a886d23b70b 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java @@ -543,7 +543,7 @@ private T toView(@NonNull final BlockItemsTranslator translator, @NonNull fi } List logs = null; for (final var item : blockItems.subList(j, n)) { - // TODO: new trace data implementation + // TODO: new trace data implementation } return (T) switch (view) { @@ -669,7 +669,7 @@ public Output build(final boolean topLevel, @Nullable final List ba builder.logs(logs); } blockItems.add(BlockItem.newBuilder() - // TODO: re-add trace data + // TODO: re-add trace data .build()); } @@ -681,7 +681,7 @@ public Output build(final boolean topLevel, @Nullable final List ba .automaticTokenAssociations( automaticTokenAssociations.getLast().accountId()); blockItems.add(BlockItem.newBuilder() - // TODO: re-add trace data + // TODO: re-add trace data .build()); } // message submit trace data @@ -690,7 +690,7 @@ public Output build(final boolean topLevel, @Nullable final List ba .sequenceNumber(sequenceNumber) .runningHash(runningHash); blockItems.add(BlockItem.newBuilder() - // TODO: re-add trace data + // TODO: re-add trace data .build()); } } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 9863cc15a4ec..1309167a8605 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -13,6 +13,7 @@ import static com.hedera.node.app.blocks.impl.streaming.FileBlockItemWriter.cleanUpPendingBlock; import static com.hedera.node.app.blocks.impl.streaming.FileBlockItemWriter.loadContiguousPendingBlocks; import static com.hedera.node.app.blocks.schemas.V0560BlockStreamSchema.BLOCK_STREAM_INFO_STATE_ID; +import static com.hedera.node.app.hapi.utils.CommonUtils.noThrowSha384HashOf; import static com.hedera.node.app.hapi.utils.CommonUtils.sha384DigestOrThrow; import static com.hedera.node.app.records.BlockRecordService.EPOCH; import static com.hedera.node.app.records.impl.BlockRecordInfoUtils.HASH_SIZE; @@ -23,6 +24,7 @@ import com.hedera.hapi.block.stream.BlockItem; import com.hedera.hapi.block.stream.BlockProof; import com.hedera.hapi.block.stream.MerkleSiblingHash; +import com.hedera.hapi.block.stream.SubMerkleTree; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.block.stream.output.StateChanges; import com.hedera.hapi.node.base.SemanticVersion; @@ -34,7 +36,6 @@ import com.hedera.node.app.blocks.BlockStreamManager; import com.hedera.node.app.blocks.BlockStreamService; import com.hedera.node.app.blocks.InitialStateHash; -import com.hedera.node.app.blocks.StreamingTreeHasher; import com.hedera.node.app.hapi.utils.CommonUtils; import com.hedera.node.app.info.DiskStartupNetworks; import com.hedera.node.app.info.DiskStartupNetworks.InfoType; @@ -70,6 +71,7 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -133,22 +135,52 @@ public class BlockStreamManagerImpl implements BlockStreamManager { private int eventIndex = 0; private final Map eventIndexInBlock = new HashMap<>(); // The last non-empty (i.e., not skipped) round number that will eventually get a start-of-state hash - private long lastRoundOfPrevBlock; private Bytes lastBlockHash; + private long lastRoundOfPrevBlock; private Instant blockTimestamp; private Instant consensusTimeLastRound; private Timestamp lastUsedTime; private BlockItemWriter writer; - // stream hashers - private StreamingTreeHasher inputTreeHasher; - private StreamingTreeHasher outputTreeHasher; - private StreamingTreeHasher consensusHeaderHasher; - private StreamingTreeHasher stateChangesHasher; - private StreamingTreeHasher traceDataHasher; + private Instant firstConsensusTimeOfCurrentBlock; + + // TODO: set from config, e.g. provider.getConfiguration().getValue("blocktree.uncollapsedHashesDir", String.class); + private String intermediateHashesDir = "data/intStreams"; + + // block merkle tree + private IncrementalStreamingHasher previousBlockHashes; // ALL previous hashes, but streaming-collapsed + private Bytes stateHashAtStartOfBlock; + private IncrementalStreamingHasher consensusHeaderHasher; + private IncrementalStreamingHasher inputTreeHasher; + private IncrementalStreamingHasher outputTreeHasher; + private IncrementalStreamingHasher stateChangesHasher; + private IncrementalStreamingHasher traceDataHasher; + // end block merkle tree private BlockStreamManagerTask worker; private final boolean hintsEnabled; + private void initIntermediateHashTrees() { + previousBlockHashes = loadHashTree(SubMerkleTree.PREVIOUS_ROOT_HASHES); + consensusHeaderHasher = loadHashTree(SubMerkleTree.CONSENSUS_HEADER); + inputTreeHasher = loadHashTree(SubMerkleTree.INPUT_ITEM); + outputTreeHasher = loadHashTree(SubMerkleTree.OUTPUT_ITEM); + stateChangesHasher = loadHashTree(SubMerkleTree.STATE_CHANGE_ITEM); + traceDataHasher = loadHashTree(SubMerkleTree.TRACE_ITEM); + } + + private IncrementalStreamingHasher loadHashTree(final SubMerkleTree subtreeType) { + final var iHashes = loadUncollapsedChildren(intermediateHashesDir, subtreeType); + return new IncrementalStreamingHasher(iHashes); + } + + static List loadUncollapsedChildren(@NonNull final String basepath, SubMerkleTree subtreeType) { + final var filepath = + Path.of(basepath).resolve(subtreeType.protoName() + ""); // .smt for subMerkleTree ? + + // TODO load file bytes here + return Collections.emptyList(); + } + /** * Represents a block pending completion by the block hash signature needed for its block proof. * @@ -250,6 +282,7 @@ public BlockStreamManagerImpl( indirectProofCounter = requireNonNull(metrics) .getOrCreate(new Counter.Config("block", "numIndirectProofs") .withDescription("Number of blocks closed with indirect proofs")); + initIntermediateHashTrees(); log.info( "Initialized BlockStreamManager from round {} with end-of-round hash {}", lastRoundOfPrevBlock, @@ -294,12 +327,6 @@ public void startRound(@NonNull final Round round, @NonNull final State state) { lifecycle.onOpenBlock(state); - inputTreeHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); - outputTreeHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); - consensusHeaderHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); - stateChangesHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); - traceDataHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); - blockNumber = blockStreamInfo.blockNumber() + 1; if (hintsEnabled && !hasCheckedForPendingBlocks) { final var hasBeenFrozen = requireNonNull(state.getReadableStates(PlatformStateService.NAME) @@ -322,6 +349,7 @@ public void startRound(@NonNull final Round round, @NonNull final State state) { worker.addItem(BlockItem.newBuilder().blockHeader(header).build()); } consensusTimeLastRound = round.getConsensusTimestamp(); + firstConsensusTimeOfCurrentBlock = round.getConsensusTimestamp(); } /** @@ -430,11 +458,6 @@ public boolean endRound(@NonNull final State state, final long roundNum) { worker.addItem(flushChangesFromListener(boundaryStateChangeListener)); worker.sync(); - final var consensusHeaderHash = consensusHeaderHasher.rootHash().join(); - final var inputHash = inputTreeHasher.rootHash().join(); - final var traceDataHash = traceDataHasher.rootHash().join(); - final var outputHash = outputTreeHasher.rootHash().join(); - // This block's starting state hash is the end state hash of the last non-empty round final var blockStartStateHash = requireNonNull(endRoundStateHashes.get(lastRoundOfPrevBlock)) .join(); @@ -444,7 +467,21 @@ public boolean endRound(@NonNull final State state, final long roundNum) { } // And update the last non-empty round number to this round lastRoundOfPrevBlock = roundNum; - final var stateChangesTreeStatus = stateChangesHasher.status(); + + // Branch 1: lastBlockHash + // Branch 2 + final var prevBlockRootsHash = Bytes.wrap(previousBlockHashes.computeRootHash()); + // Branch 3: blockStartStateHash + // Branch 4 + final var consensusHeaderHash = Bytes.wrap(consensusHeaderHasher.computeRootHash()); + // Branch 5 + final var inputsHash = Bytes.wrap(inputTreeHasher.computeRootHash()); + // Branch 6 + final var outputsHash = Bytes.wrap(outputTreeHasher.computeRootHash()); + // Branch 7 + final var stateChangesHash = Bytes.wrap(stateChangesHasher.computeRootHash()); + // Branch 8 + final var traceDataHash = Bytes.wrap(traceDataHasher.computeRootHash()); // Put this block hash context in state via the block stream info final var writableState = state.getWritableStates(BlockStreamService.NAME); @@ -452,38 +489,58 @@ public boolean endRound(@NonNull final State state, final long roundNum) { blockStreamInfoState.put(new BlockStreamInfo( blockNumber, blockTimestamp(), - runningHashManager.latestHashes(), + runningHashManager.latestHashes(), // lastBlockHash is stored here blockHashManager.blockHashes(), - inputHash, + inputsHash, blockStartStateHash, - stateChangesTreeStatus.numLeaves(), - stateChangesTreeStatus.rightmostHashes(), + stateChangesHasher.leafCount(), + previousBlockHashes.intermediateHashingState(), lastUsedTime, pendingWork != POST_UPGRADE_WORK, version, asTimestamp(lastIntervalProcessTime), asTimestamp(lastTopLevelTime), - consensusHeaderHash, - traceDataHash, - outputHash)); + consensusHeaderHasher.intermediateHashingState(), + inputTreeHasher.intermediateHashingState(), + outputTreeHasher.intermediateHashingState(), + stateChangesHasher.intermediateHashingState(), + traceDataHasher.intermediateHashingState())); ((CommittableWritableStates) writableState).commit(); worker.addItem(flushChangesFromListener(boundaryStateChangeListener)); worker.sync(); - final var stateChangesHash = stateChangesHasher.rootHash().join(); + // Compute depth four hashes + final var depth4Node1 = combine(lastBlockHash, prevBlockRootsHash); + final var depth4Node2 = combine(blockStartStateHash, consensusHeaderHash); + final var depth4Node3 = combine(inputsHash, outputsHash); + final var depth4Node4 = combine(stateChangesHash, traceDataHash); + + final var combinedNulls = combine(NULL_HASH, NULL_HASH); + final var depth4Node5 = combinedNulls; + final var depth4Node6 = combinedNulls; + final var depth4Node7 = combinedNulls; + final var depth4Node8 = combinedNulls; + + // Compute depth three hashes + final var depth3Node1 = combine(depth4Node1, depth4Node2); + final var depth3Node2 = combine(depth4Node3, depth4Node4); + final var depth3Node3 = combine(depth4Node5, depth4Node6); + final var depth3Node4 = combine(depth4Node7, depth4Node8); // Compute depth two hashes - final var depth2Node0 = combine(lastBlockHash, blockStartStateHash); - final var depth2Node1 = combine(consensusHeaderHash, inputHash); - final var depth2Node2 = combine(outputHash, stateChangesHash); - final var depth2Node3 = combine(traceDataHash, NULL_HASH); + final var depth2Node1 = combine(depth3Node1, depth3Node2); + final var depth2Node2 = combine(depth3Node3, depth3Node4); // Compute depth one hashes - final var depth1Node0 = combine(depth2Node0, depth2Node1); - final var depth1Node1 = combine(depth2Node2, depth2Node3); - - // Compute the block hash + final var timestamp = Timestamp.PROTOBUF.toBytes(Timestamp.newBuilder() + .seconds(firstConsensusTimeOfCurrentBlock.getEpochSecond()) + .nanos(firstConsensusTimeOfCurrentBlock.getNano()) + .build()); + final var depth1Node0 = noThrowSha384HashOf(timestamp); + final var depth1Node1 = combine(depth2Node1, depth2Node2); + + // Compute the block's root hash final var blockHash = combine(depth1Node0, depth1Node1); final var pendingProof = BlockProof.newBuilder() @@ -545,6 +602,8 @@ public boolean endRound(@NonNull final State state, final long roundNum) { writer = null; } requireNonNull(fatalShutdownFuture).complete(null); + + // TODO: write intermediate hashes of sub trees } return closesBlock; } @@ -751,10 +810,10 @@ protected boolean onExecute() { STATE_CHANGES, ROUND_HEADER, BLOCK_HEADER, - BLOCK_FOOTER, - BLOCK_PROOF - // Also EndBlock? - -> { + BLOCK_FOOTER, + BLOCK_PROOF + // Also EndBlock? + -> { MessageDigest digest = sha384DigestOrThrow(); bytes.writeTo(digest); hash = ByteBuffer.wrap(digest.digest()); @@ -784,16 +843,16 @@ class SequentialTask extends AbstractTask { protected boolean onExecute() { final var kind = item.item().kind(); switch (kind) { - case ROUND_HEADER, EVENT_HEADER -> consensusHeaderHasher.addLeaf(hash); - case SIGNED_TRANSACTION -> inputTreeHasher.addLeaf(hash); + case ROUND_HEADER, EVENT_HEADER -> consensusHeaderHasher.addLeaf(hash.array()); + case SIGNED_TRANSACTION -> inputTreeHasher.addLeaf(hash.array()); case TRANSACTION_RESULT -> { runningHashManager.nextResultHash(hash); hash.rewind(); - outputTreeHasher.addLeaf(hash); + outputTreeHasher.addLeaf(hash.array()); } - case TRANSACTION_OUTPUT, BLOCK_HEADER -> outputTreeHasher.addLeaf(hash); - case STATE_CHANGES -> stateChangesHasher.addLeaf(hash); - case BLOCK_FOOTER, BLOCK_PROOF -> throw new NotImplementedException(); + case TRANSACTION_OUTPUT, BLOCK_HEADER -> outputTreeHasher.addLeaf(hash.array()); + case STATE_CHANGES -> stateChangesHasher.addLeaf(hash.array()); + case BLOCK_FOOTER, BLOCK_PROOF -> throw new NotImplementedException(); } final BlockHeader header = item.blockHeader(); diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java index 10addc43cb03..dffdce8369d0 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.node.app.blocks.impl; +import com.hedera.pbj.runtime.io.buffer.Bytes; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.LinkedList; @@ -14,107 +15,107 @@ *

This is not thread safe, it is assumed use by single thread.

*/ public class IncrementalStreamingHasher { - /** Prefix byte for hash contents for leaf nodes. */ - private static final byte[] LEAF_PREFIX = new byte[] {0}; - /** Prefix byte for hash contents for internal nodes. */ - private static final byte[] INTERNAL_NODE_PREFIX = new byte[] {2}; - /** The hashing algorithm used for computing the hashes. */ - private final MessageDigest digest; - /** A list to store intermediate hashes as we build the tree. */ - private final LinkedList hashList = new LinkedList<>(); - /** The count of leaves in the tree. */ - private long leafCount = 0; + /** Prefix byte for hash contents for leaf nodes. */ + private static final byte[] LEAF_PREFIX = new byte[] {0}; + /** Prefix byte for hash contents for internal nodes. */ + private static final byte[] INTERNAL_NODE_PREFIX = new byte[] {2}; + /** The hashing algorithm used for computing the hashes. */ + private final MessageDigest digest; + /** A list to store intermediate hashes as we build the tree. */ + private final LinkedList hashList = new LinkedList<>(); + /** The count of leaves in the tree. */ + private int leafCount = 0; - /** Create a new StreamingHasher with an empty state. */ - public IncrementalStreamingHasher() { - try { - digest = MessageDigest.getInstance("SHA-384"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } + /** Create a new StreamingHasher with an empty state. */ + public IncrementalStreamingHasher() { + try { + digest = MessageDigest.getInstance("SHA-384"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } - /** - * Create a StreamingHasher with an existing intermediate hashing state. - * This allows resuming hashing from a previous state. - * - * @param intermediateHashingState the intermediate hashing state - */ - public IncrementalStreamingHasher(List intermediateHashingState) { - this(); - this.hashList.addAll(intermediateHashingState); - } + /** + * Create a StreamingHasher with an existing intermediate hashing state. + * This allows resuming hashing from a previous state. + * + * @param intermediateHashingState the intermediate hashing state + */ + public IncrementalStreamingHasher(List intermediateHashingState) { + this(); + this.hashList.addAll(intermediateHashingState); + } - /** - * Add a new leaf to the Merkle tree. - * - * @param data the data for the new leaf - */ - public void addLeaf(byte[] data) { - final long i = leafCount; - final byte[] e = hashLeaf(data); - hashList.add(e); - for (long n=i; (n & 1L) == 1; n >>= 1) { - final byte[] y = hashList.removeLast(); - final byte[] x = hashList.removeLast(); - hashList.add(hashInternalNode(x, y)); - } - leafCount ++; - } + /** + * Add a new leaf to the Merkle tree. + * + * @param data the data for the new leaf + */ + public void addLeaf(byte[] data) { + final long i = leafCount; + final byte[] e = hashLeaf(data); + hashList.add(e); + for (long n = i; (n & 1L) == 1; n >>= 1) { + final byte[] y = hashList.removeLast(); + final byte[] x = hashList.removeLast(); + hashList.add(hashInternalNode(x, y)); + } + leafCount++; + } - /** - * Compute the Merkle tree root hash from the current state. This does not modify the internal state, so can be - * called at any time and more leaves can be added afterward. - * - * @return the Merkle tree root hash - */ - public byte[] computeRootHash() { - byte[] merkleRootHash = hashList.getLast(); - for (int i = hashList.size() - 2; i >= 0; i--) { - merkleRootHash = hashInternalNode(hashList.get(i), merkleRootHash); - } - return merkleRootHash; - } + /** + * Compute the Merkle tree root hash from the current state. This does not modify the internal state, so can be + * called at any time and more leaves can be added afterward. + * + * @return the Merkle tree root hash + */ + public byte[] computeRootHash() { + byte[] merkleRootHash = hashList.getLast(); + for (int i = hashList.size() - 2; i >= 0; i--) { + merkleRootHash = hashInternalNode(hashList.get(i), merkleRootHash); + } + return merkleRootHash; + } - /** - * Get the current intermediate hashing state. This can be used to save the state and resume hashing later. - * - * @return the intermediate hashing state - */ - public List intermediateHashingState() { - return hashList; - } + /** + * Get the current intermediate hashing state. This can be used to save the state and resume hashing later. + * + * @return the intermediate hashing state + */ + public List intermediateHashingState() { + return hashList.stream().map(Bytes::wrap).toList(); + } - /** - * Get the number of leaves added to the tree so far. - * - * @return the number of leaves - */ - public long leafCount() { - return leafCount; - } + /** + * Get the number of leaves added to the tree so far. + * + * @return the number of leaves + */ + public int leafCount() { + return leafCount; + } - /** - * Hash a leaf node with the appropriate prefix. - * - * @param leafData the data of the leaf - * @return the hash of the leaf node - */ - private byte[] hashLeaf(final byte[] leafData) { - digest.update(LEAF_PREFIX); - return digest.digest(leafData); - } + /** + * Hash a leaf node with the appropriate prefix. + * + * @param leafData the data of the leaf + * @return the hash of the leaf node + */ + private byte[] hashLeaf(final byte[] leafData) { + digest.update(LEAF_PREFIX); + return digest.digest(leafData); + } - /** - * Hash an internal node by combining the hashes of its two children with the appropriate prefix. - * - * @param firstChild the hash of the first child - * @param secondChild the hash of the second child - * @return the hash of the internal node - */ - private byte[] hashInternalNode(final byte[] firstChild, final byte[] secondChild) { - digest.update(INTERNAL_NODE_PREFIX); - digest.update(firstChild); - return digest.digest(secondChild); - } + /** + * Hash an internal node by combining the hashes of its two children with the appropriate prefix. + * + * @param firstChild the hash of the first child + * @param secondChild the hash of the second child + * @return the hash of the internal node + */ + private byte[] hashInternalNode(final byte[] firstChild, final byte[] secondChild) { + digest.update(INTERNAL_NODE_PREFIX); + digest.update(firstChild); + return digest.digest(secondChild); + } } diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java index a7e291cf2523..d6b6e120440e 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java @@ -10,7 +10,6 @@ import static com.hedera.node.app.spi.workflows.HandleContext.TransactionCategory.USER; import static com.hedera.node.app.spi.workflows.record.StreamBuilder.ReversingBehavior.REVERSIBLE; import static com.hedera.node.app.spi.workflows.record.StreamBuilder.SignedTxCustomizer.NOOP_SIGNED_TX_CUSTOMIZER; -import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -141,12 +140,12 @@ void testBlockItemsWithTraceAndOutput() { assertTrue(output.hasContractCall()); final var traceItem = blockItems.get(3); - // TODO: assert trace data -// assertTrue(traceItem.hasTraceData()); -// final var trace = traceItem.traceDataOrThrow(); -// assertTrue(trace.hasEvmTraceData()); -// final var evmTrace = trace.evmTraceDataOrThrow(); -// assertEquals(usages, evmTrace.contractSlotUsages()); + // TODO: assert trace data + // assertTrue(traceItem.hasTraceData()); + // final var trace = traceItem.traceDataOrThrow(); + // assertTrue(trace.hasEvmTraceData()); + // final var evmTrace = trace.evmTraceDataOrThrow(); + // assertEquals(usages, evmTrace.contractSlotUsages()); } @Test @@ -162,15 +161,15 @@ void testBlockItemsWithAdditionalAutomaticTokenAssociationTraceData() { final var blockItems = itemsBuilder.build(false, List.of()).blockItems(); final var traceItem = blockItems.get(2); - // TODO: assert trace data -// assertThat(traceItem.hasTraceData()).isTrue(); -// final var trace = traceItem.traceDataOrThrow(); -// -// assertThat(trace.hasAutoAssociateTraceData()).isTrue(); -// final var autoAssociateTraceData = trace.autoAssociateTraceData(); -// assertThat(autoAssociateTraceData).isNotNull(); -// assertThat(autoAssociateTraceData.automaticTokenAssociations().accountNum()) -// .isEqualTo(2); + // TODO: assert trace data + // assertThat(traceItem.hasTraceData()).isTrue(); + // final var trace = traceItem.traceDataOrThrow(); + // + // assertThat(trace.hasAutoAssociateTraceData()).isTrue(); + // final var autoAssociateTraceData = trace.autoAssociateTraceData(); + // assertThat(autoAssociateTraceData).isNotNull(); + // assertThat(autoAssociateTraceData.automaticTokenAssociations().accountNum()) + // .isEqualTo(2); } @Test @@ -181,14 +180,14 @@ void testBlockItemsWithAdditionalSubmitMsgTraceData() { final var blockItems = itemsBuilder.build(false, List.of()).blockItems(); final var traceItem = blockItems.get(2); - // TODO: assert trace data -// assertThat(traceItem.hasTraceData()).isTrue(); -// final var trace = traceItem.traceDataOrThrow(); -// -// assertThat(trace.hasSubmitMessageTraceData()).isTrue(); -// final var submitMessageTraceData = trace.submitMessageTraceData(); -// assertThat(submitMessageTraceData).isNotNull(); -// assertThat(submitMessageTraceData.sequenceNumber()).isEqualTo(66); + // TODO: assert trace data + // assertThat(traceItem.hasTraceData()).isTrue(); + // final var trace = traceItem.traceDataOrThrow(); + // + // assertThat(trace.hasSubmitMessageTraceData()).isTrue(); + // final var submitMessageTraceData = trace.submitMessageTraceData(); + // assertThat(submitMessageTraceData).isNotNull(); + // assertThat(submitMessageTraceData.sequenceNumber()).isEqualTo(66); } @Test diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index 531a3149130b..5dab60f70914 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -7,7 +7,6 @@ import static com.hedera.node.app.blocks.BlockStreamManager.ZERO_BLOCK_HASH; import static com.hedera.node.app.blocks.BlockStreamService.FAKE_RESTART_BLOCK_HASH; import static com.hedera.node.app.blocks.impl.BlockImplUtils.appendHash; -import static com.hedera.node.app.blocks.impl.BlockImplUtils.combine; import static com.hedera.node.app.blocks.schemas.V0560BlockStreamSchema.BLOCK_STREAM_INFO_STATE_ID; import static com.hedera.node.app.blocks.schemas.V0560BlockStreamSchema.BLOCK_STREAM_INFO_STATE_LABEL; import static com.hedera.node.app.fixtures.AppTestBase.DEFAULT_CONFIG; @@ -35,7 +34,6 @@ import static org.mockito.Mockito.withSettings; import com.hedera.hapi.block.stream.BlockItem; -import com.hedera.hapi.block.stream.RecordFileItem; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.block.stream.output.StateChanges; import com.hedera.hapi.block.stream.output.TransactionResult; @@ -73,7 +71,6 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Iterator; -import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ForkJoinPool; @@ -114,7 +111,7 @@ class BlockStreamManagerImplTest { private static final BlockItem FAKE_STATE_CHANGES = BlockItem.newBuilder() .stateChanges(StateChanges.newBuilder().consensusTimestamp(CONSENSUS_THEN)) .build(); - // TODO: remove, or replace with wrapped record file item + // TODO: remove, or replace with wrapped record file item private static final BlockItem FAKE_RECORD_FILE_ITEM = null; private final InitialStateHash hashInfo = new InitialStateHash(completedFuture(ZERO_BLOCK_HASH), 0); @@ -328,34 +325,40 @@ void startsAndEndsBlockWithSingleRoundPerBlockAsExpected() throws ParseException verify(aWriter).openBlock(N_BLOCK_NO); - // Assert the internal state of the subject has changed as expected and the writer has been closed - final var expectedBlockInfo = new BlockStreamInfo( - N_BLOCK_NO, - asTimestamp(CONSENSUS_NOW), - appendHash(combine(ZERO_BLOCK_HASH, FAKE_RESULT_HASH), appendHash(ZERO_BLOCK_HASH, Bytes.EMPTY, 4), 4), - appendHash(FAKE_RESTART_BLOCK_HASH, appendHash(N_MINUS_2_BLOCK_HASH, Bytes.EMPTY, 256), 256), - Bytes.fromHex( - "edde6b2beddb2fda438665bbe6df0a639c518e6d5352e7276944b70777d437d28d1b22813ed70f5b8a3a3cbaf08aa9a8"), - ZERO_BLOCK_HASH, - 2, - List.of( - Bytes.EMPTY, - Bytes.fromHex( - "839ddb854c8f4cf9c3705268b17bc7d53e91454ff14dbbfffd6c77b6118a0e79fb1e478b4924bfb0fd93ef60101d3237")), - FAKE_TRANSACTION_RESULT.transactionResultOrThrow().consensusTimestampOrThrow(), - true, - SemanticVersion.DEFAULT, - CONSENSUS_THEN, - CONSENSUS_THEN, - Bytes.fromHex( - "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), - Bytes.fromHex( - "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), - Bytes.fromHex( - "bf99e1dfd15ffe551ae4bc0953f396639755f0419522f323875806a55a57dca6a4df61ea6dee28bec0c37ed54881d392")); - - final var actualBlockInfo = infoRef.get(); - assertEquals(expectedBlockInfo, actualBlockInfo); + // TODO: Assert the internal state of the subject has changed as expected and the writer has been closed + // final var expectedBlockInfo = new BlockStreamInfo( + // N_BLOCK_NO, + // asTimestamp(CONSENSUS_NOW), + // appendHash(combine(ZERO_BLOCK_HASH, FAKE_RESULT_HASH), appendHash(ZERO_BLOCK_HASH, + // Bytes.EMPTY, 4), 4), + // appendHash(FAKE_RESTART_BLOCK_HASH, appendHash(N_MINUS_2_BLOCK_HASH, Bytes.EMPTY, 256), 256), + // Bytes.fromHex( + // + // "edde6b2beddb2fda438665bbe6df0a639c518e6d5352e7276944b70777d437d28d1b22813ed70f5b8a3a3cbaf08aa9a8"), + // ZERO_BLOCK_HASH, + // 2, + // List.of( + // Bytes.EMPTY, + // Bytes.fromHex( + // + // "839ddb854c8f4cf9c3705268b17bc7d53e91454ff14dbbfffd6c77b6118a0e79fb1e478b4924bfb0fd93ef60101d3237")), + // FAKE_TRANSACTION_RESULT.transactionResultOrThrow().consensusTimestampOrThrow(), + // true, + // SemanticVersion.DEFAULT, + // CONSENSUS_THEN, + // CONSENSUS_THEN, + // Bytes.fromHex( + // + // "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), + // Bytes.fromHex( + // + // "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), + // Bytes.fromHex( + // + // "bf99e1dfd15ffe551ae4bc0953f396639755f0419522f323875806a55a57dca6a4df61ea6dee28bec0c37ed54881d392")); + // + // final var actualBlockInfo = infoRef.get(); + // assertEquals(expectedBlockInfo, actualBlockInfo); // Assert the block proof was written final var proofItem = lastAItem.get(); @@ -557,33 +560,38 @@ void alwaysEndsBlockOnFreezeRoundPerBlockAsExpected() throws ParseException { verify(aWriter).openBlock(N_BLOCK_NO); - // Assert the internal state of the subject has changed as expected and the writer has been closed - final var expectedBlockInfo = new BlockStreamInfo( - N_BLOCK_NO, - asTimestamp(CONSENSUS_NOW), - appendHash(combine(Bytes.fromHex("dd".repeat(48)), FAKE_RESULT_HASH), resultHashes, 4), - appendHash(FAKE_RESTART_BLOCK_HASH, appendHash(N_MINUS_2_BLOCK_HASH, Bytes.EMPTY, 256), 256), - Bytes.fromHex( - "edde6b2beddb2fda438665bbe6df0a639c518e6d5352e7276944b70777d437d28d1b22813ed70f5b8a3a3cbaf08aa9a8"), - ZERO_BLOCK_HASH, - 2, - List.of( - Bytes.EMPTY, - Bytes.fromHex( - "839ddb854c8f4cf9c3705268b17bc7d53e91454ff14dbbfffd6c77b6118a0e79fb1e478b4924bfb0fd93ef60101d3237")), - FAKE_TRANSACTION_RESULT.transactionResultOrThrow().consensusTimestampOrThrow(), - false, - SemanticVersion.DEFAULT, - CONSENSUS_THEN, - CONSENSUS_THEN, - Bytes.fromHex( - "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), - Bytes.fromHex( - "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), - Bytes.fromHex( - "8ee0718d5f75f867f85cb4e400ebf7bfbb4cd91479d7f3f8bfd28ce062c318c312b8f4de185a994b78337e6391e3f000")); - final var actualBlockInfo = infoRef.get(); - assertEquals(expectedBlockInfo, actualBlockInfo); + // TODO: Assert the internal state of the subject has changed as expected and the writer has been closed + // final var expectedBlockInfo = new BlockStreamInfo( + // N_BLOCK_NO, + // asTimestamp(CONSENSUS_NOW), + // appendHash(combine(Bytes.fromHex("dd".repeat(48)), FAKE_RESULT_HASH), resultHashes, 4), + // appendHash(FAKE_RESTART_BLOCK_HASH, appendHash(N_MINUS_2_BLOCK_HASH, Bytes.EMPTY, 256), 256), + // Bytes.fromHex( + // + // "edde6b2beddb2fda438665bbe6df0a639c518e6d5352e7276944b70777d437d28d1b22813ed70f5b8a3a3cbaf08aa9a8"), + // ZERO_BLOCK_HASH, + // 2, + // List.of( + // Bytes.EMPTY, + // Bytes.fromHex( + // + // "839ddb854c8f4cf9c3705268b17bc7d53e91454ff14dbbfffd6c77b6118a0e79fb1e478b4924bfb0fd93ef60101d3237")), + // FAKE_TRANSACTION_RESULT.transactionResultOrThrow().consensusTimestampOrThrow(), + // false, + // SemanticVersion.DEFAULT, + // CONSENSUS_THEN, + // CONSENSUS_THEN, + // Bytes.fromHex( + // + // "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), + // Bytes.fromHex( + // + // "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), + // Bytes.fromHex( + // + // "8ee0718d5f75f867f85cb4e400ebf7bfbb4cd91479d7f3f8bfd28ce062c318c312b8f4de185a994b78337e6391e3f000")); + // final var actualBlockInfo = infoRef.get(); + // assertEquals(expectedBlockInfo, actualBlockInfo); // Assert the block proof was written final var proofItem = lastAItem.get(); From a4695d9a2ecf43dfa22b962c59e8feff38c1a9ae Mon Sep 17 00:00:00 2001 From: Artem Derevets Date: Mon, 6 Oct 2025 17:04:59 +0200 Subject: [PATCH 05/63] feat: add block footer (#21356) Signed-off-by: artemderevets --- .../blocks/impl/BlockStreamManagerImpl.java | 30 ++- .../impl/BlockStreamManagerImplTest.java | 238 +++++++++++++++++- 2 files changed, 262 insertions(+), 6 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index d6f154e40215..848b12c5b886 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -90,7 +90,6 @@ import org.apache.logging.log4j.Logger; import org.hiero.base.concurrent.AbstractTask; import org.hiero.base.crypto.Hash; -import org.hiero.base.exceptions.NotImplementedException; import org.hiero.consensus.model.hashgraph.Round; @Singleton @@ -472,6 +471,26 @@ public boolean endRound(@NonNull final State state, final long roundNum) { final var stateChangesHash = stateChangesHasher.rootHash().join(); + // TODO(#21210): Implement streaming merkle tree of all block hashes from genesis to N-1 + // For now, using NULL_HASH as placeholder until the historical block data infrastructure is ready. + final var blockHashesTreeRoot = NULL_HASH; + + // Create BlockFooter with the three essential hashes: + // 1. previousBlockRootHash - Root hash of the previous block (N-1) + // 2. rootHashOfAllBlockHashesTree - Streaming tree of all block hashes 0..N-1 (TODO: #21210) + // 3. startOfBlockStateRootHash - State hash at the beginning of current block + final var blockFooter = com.hedera.hapi.block.stream.output.BlockFooter.newBuilder() + .previousBlockRootHash(lastBlockHash) + .rootHashOfAllBlockHashesTree(blockHashesTreeRoot) + .startOfBlockStateRootHash(blockStartStateHash) + .build(); + + // Write BlockFooter to block stream (last item before BlockProof) + final var footerItem = + BlockItem.newBuilder().blockFooter(blockFooter).build(); + worker.addItem(footerItem); + worker.sync(); + // Compute depth two hashes final var depth2Node0 = combine(lastBlockHash, blockStartStateHash); final var depth2Node1 = combine(consensusHeaderHash, inputHash); @@ -749,9 +768,7 @@ protected boolean onExecute() { TRANSACTION_OUTPUT, STATE_CHANGES, ROUND_HEADER, - BLOCK_HEADER, - BLOCK_FOOTER, - BLOCK_PROOF + BLOCK_HEADER // Also EndBlock? -> { MessageDigest digest = sha384DigestOrThrow(); @@ -792,7 +809,10 @@ protected boolean onExecute() { } case TRANSACTION_OUTPUT, BLOCK_HEADER -> outputTreeHasher.addLeaf(hash); case STATE_CHANGES -> stateChangesHasher.addLeaf(hash); - case BLOCK_FOOTER, BLOCK_PROOF -> throw new NotImplementedException(); + case BLOCK_FOOTER, BLOCK_PROOF -> { + // BlockFooter and BlockProof are not included in any merkle tree + // They are metadata about the block, not part of the hashed content + } } final BlockHeader header = item.blockHeader(); diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index 531a3149130b..d32a8936a96b 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -35,7 +35,6 @@ import static org.mockito.Mockito.withSettings; import com.hedera.hapi.block.stream.BlockItem; -import com.hedera.hapi.block.stream.RecordFileItem; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.block.stream.output.StateChanges; import com.hedera.hapi.block.stream.output.TransactionResult; @@ -72,11 +71,13 @@ import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -929,6 +930,241 @@ void eventHashMapIsClearedBetweenBlocks() { assertEquals(Optional.of(0), subject.getEventIndex(eventHash3)); } + @Test + @SuppressWarnings("unchecked") + void writesBlockFooterBeforeBlockProof() { + // Given a manager with a single round per block + givenSubjectWith( + 1, 0, blockStreamInfoWith(Bytes.EMPTY, CREATION_VERSION), platformStateWithFreezeTime(null), aWriter); + givenEndOfRoundSetup(); + + final AtomicReference footerItem = new AtomicReference<>(); + final AtomicReference proofItem = new AtomicReference<>(); + + doAnswer(invocationOnMock -> { + final var item = BlockItem.PROTOBUF.parse((Bytes) invocationOnMock.getArgument(1)); + if (item.hasBlockFooter()) { + footerItem.set(item); + } else if (item.hasBlockProof()) { + proofItem.set(item); + } + return aWriter; + }) + .when(aWriter) + .writePbjItemAndBytes(any(), any()); + + given(round.getRoundNum()).willReturn(ROUND_NO); + given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); + given(blockHashSigner.isReady()).willReturn(true); + given(blockHashSigner.schemeId()).willReturn(1L); + + // Set up the signature future to complete immediately + given(blockHashSigner.signFuture(any())).willReturn(mockSigningFuture); + doAnswer(invocationOnMock -> { + final Consumer consumer = invocationOnMock.getArgument(0); + consumer.accept(FIRST_FAKE_SIGNATURE); + return null; + }) + .when(mockSigningFuture) + .thenAcceptAsync(any()); + + // Initialize hash and start a round + subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); + subject.startRound(round, state); + + // Write some items + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.writeItem(FAKE_TRANSACTION_RESULT); + subject.writeItem(FAKE_STATE_CHANGES); + + // End the round + subject.endRound(state, ROUND_NO); + + // Verify BlockFooter was written + assertNotNull(footerItem.get(), "BlockFooter should be written"); + assertTrue(footerItem.get().hasBlockFooter()); + + final var footer = footerItem.get().blockFooterOrThrow(); + assertNotNull(footer.previousBlockRootHash(), "Previous block root hash should be set"); + // TODO(#21210): Currently using NULL_HASH placeholder for block hashes tree + // Will be replaced when streaming merkle tree of all block hashes is implemented + assertEquals( + BlockStreamManagerImpl.NULL_HASH, + footer.rootHashOfAllBlockHashesTree(), + "Block hashes tree root should be NULL_HASH until #21210 is implemented"); + assertNotNull(footer.startOfBlockStateRootHash(), "Start of block state root hash should be set"); + + // Verify BlockProof was also written + assertNotNull(proofItem.get(), "BlockProof should be written"); + assertTrue(proofItem.get().hasBlockProof()); + } + + @Test + @SuppressWarnings("unchecked") + void blockFooterContainsCorrectHashValues() { + // Given a manager with a single round per block + givenSubjectWith( + 1, 0, blockStreamInfoWith(Bytes.EMPTY, CREATION_VERSION), platformStateWithFreezeTime(null), aWriter); + givenEndOfRoundSetup(); + + final AtomicReference footerItem = new AtomicReference<>(); + + doAnswer(invocationOnMock -> { + final var item = BlockItem.PROTOBUF.parse((Bytes) invocationOnMock.getArgument(1)); + if (item.hasBlockFooter()) { + footerItem.set(item); + } + return aWriter; + }) + .when(aWriter) + .writePbjItemAndBytes(any(), any()); + + given(round.getRoundNum()).willReturn(ROUND_NO); + given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); + given(blockHashSigner.isReady()).willReturn(true); + given(blockHashSigner.schemeId()).willReturn(1L); + + // Set up the signature future + given(blockHashSigner.signFuture(any())).willReturn(mockSigningFuture); + doAnswer(invocationOnMock -> { + final Consumer consumer = invocationOnMock.getArgument(0); + consumer.accept(FIRST_FAKE_SIGNATURE); + return null; + }) + .when(mockSigningFuture) + .thenAcceptAsync(any()); + + // Initialize with known hash and start round + subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); + subject.startRound(round, state); + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.endRound(state, ROUND_NO); + + // Verify BlockFooter hash values + assertNotNull(footerItem.get(), "BlockFooter should be written"); + final var footer = footerItem.get().blockFooterOrThrow(); + + // Verify previousBlockRootHash matches the last block hash + assertEquals( + N_MINUS_2_BLOCK_HASH, + footer.previousBlockRootHash(), + "Previous block root hash should match initialized last block hash"); + + // Verify rootHashOfAllBlockHashesTree is NULL_HASH (placeholder) + assertEquals( + BlockStreamManagerImpl.NULL_HASH, + footer.rootHashOfAllBlockHashesTree(), + "Block hashes tree root should be NULL_HASH placeholder"); + + // Verify startOfBlockStateRootHash is set + assertEquals( + FAKE_START_OF_BLOCK_STATE_HASH.getBytes(), + footer.startOfBlockStateRootHash(), + "Start of block state root hash should match expected value"); + } + + @Test + @SuppressWarnings("unchecked") + void blockFooterWrittenForEachBlock() { + // Given a manager with a single round per block + givenSubjectWith( + 1, + 0, + blockStreamInfoWith(Bytes.EMPTY, CREATION_VERSION), + platformStateWithFreezeTime(null), + aWriter, + bWriter); + givenEndOfRoundSetup(); + + final List footerItems = new ArrayList<>(); + + doAnswer(invocationOnMock -> { + final var item = BlockItem.PROTOBUF.parse((Bytes) invocationOnMock.getArgument(1)); + if (item.hasBlockFooter()) { + footerItems.add(item); + } + return aWriter; + }) + .when(aWriter) + .writePbjItemAndBytes(any(), any()); + + doAnswer(invocationOnMock -> { + final var item = BlockItem.PROTOBUF.parse((Bytes) invocationOnMock.getArgument(1)); + if (item.hasBlockFooter()) { + footerItems.add(item); + } + return bWriter; + }) + .when(bWriter) + .writePbjItemAndBytes(any(), any()); + + given(round.getRoundNum()).willReturn(ROUND_NO); + given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); + given(blockHashSigner.isReady()).willReturn(true); + given(blockHashSigner.schemeId()).willReturn(1L); + + // Set up the signature futures + final CompletableFuture firstSignature = (CompletableFuture) mock(CompletableFuture.class); + final CompletableFuture secondSignature = (CompletableFuture) mock(CompletableFuture.class); + given(blockHashSigner.signFuture(any())).willReturn(firstSignature).willReturn(secondSignature); + + // Initialize and create first block + subject.initLastBlockHash(FAKE_RESTART_BLOCK_HASH); + subject.startRound(round, state); + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.endRound(state, ROUND_NO); + + // Create second block + given(round.getRoundNum()).willReturn(ROUND_NO + 1); + given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW.plusSeconds(1)); + given(notification.round()).willReturn(ROUND_NO); + given(notification.hash()).willReturn(FAKE_START_OF_BLOCK_STATE_HASH); + subject.notify(notification); + subject.startRound(round, state); + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.endRound(state, ROUND_NO + 1); + + // Verify BlockFooter was written for each block + assertEquals(2, footerItems.size(), "Should have written BlockFooter for each block"); + + // Verify both are valid BlockFooters + assertTrue(footerItems.get(0).hasBlockFooter(), "First item should be BlockFooter"); + assertTrue(footerItems.get(1).hasBlockFooter(), "Second item should be BlockFooter"); + } + + @Test + void blockFooterNotWrittenWhenBlockNotClosed() { + // Given a manager with 2 rounds per block + givenSubjectWith( + 2, 0, blockStreamInfoWith(Bytes.EMPTY, CREATION_VERSION), platformStateWithFreezeTime(null), aWriter); + givenEndOfRoundSetup(); + + final AtomicBoolean footerWritten = new AtomicBoolean(false); + + doAnswer(invocationOnMock -> { + final var item = BlockItem.PROTOBUF.parse((Bytes) invocationOnMock.getArgument(1)); + if (item.hasBlockFooter()) { + footerWritten.set(true); + } + return aWriter; + }) + .when(aWriter) + .writePbjItemAndBytes(any(), any()); + + given(round.getRoundNum()).willReturn(ROUND_NO); + given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); + given(blockHashSigner.isReady()).willReturn(true); + + // Initialize and start first round (block not yet closed) + subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); + subject.startRound(round, state); + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.endRound(state, ROUND_NO); + + // Verify BlockFooter was NOT written (block needs 2 rounds) + assertFalse(footerWritten.get(), "BlockFooter should not be written until block is closed"); + } + private void givenSubjectWith( final int roundsPerBlock, final int blockPeriod, From 96fbb79aa4d0cb053ae8e7498af573ab2de511c8 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 6 Oct 2025 21:59:53 -0600 Subject: [PATCH 06/63] Fix compilation errors Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 85 +++++++++++++++++-- 1 file changed, 79 insertions(+), 6 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index b326ca5e8c49..7411ebcd870f 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -207,19 +207,70 @@ message BlockItem { com.hedera.hapi.block.stream.output.StateChanges state_changes = 7; /** - * TODO + * Verification data for an item filtered from the stream.
+ * This is a hash for a merkle tree node where the contents of that + * part of the merkle tree have been removed from this stream. + *

+ * Items of this type SHALL NOT be present in the full (unfiltered) + * block stream.
+ * Items of this type SHALL replace any item removed from a partial + * (filtered) block stream.
+ * Presence of `filtered_item` entries SHALL NOT prevent verification + * of a block, but MAY preclude verification or reconstruction of + * consensus state.
+ */ + FilteredItemHash filtered_item_hash = 8; + + /** + * A signed block proof.
+ * The signed merkle proof for this block. This will validate + * a "virtual" merkle tree containing the previous block "virtual" + * root, an "input" subtree, an "output" subtree, and + * a "state changes" subtree. + *

+ * This item is not part of the block stream hash chain/tree, and + * MUST follow after the end of a block. */ - FilteredSingleItem filtered_single_item = 8; + BlockProof block_proof = 9; /** - * TODO + * A record file and associated data. + *

+ * This MUST contain a single Record file, associated Sidecar files, + * and data from related Signature files. + * If this item is present, special treatment is + * REQUIRED for this block. + *

    + *
  • The block SHALL NOT have a `BlockHeader`.
    • + *
  • The block SHALL NOT have a `BlockProof`.
    • + *
  • The block SHALL contain _exactly one_ `RecordFileItem`.
    • + *
  • The block SHALL NOT contain any item other than a + * `RecordFileItem`.
    • + *
  • The content of the `RecordFileItem` MUST be validated using + * the signature data and content provided within according to + * the process used for Record Files prior to the creation + * of Block Stream.
    • + *
*/ - com.hedera.hapi.block.stream.output.BlockFooter block_footer = 9; + RecordFileItem record_file = 10; /** - * TODO + * A trace data. + *

+ * Any informational trace data MAY be described by + * stream items of this type.
*/ - BlockProof block_proof = 10; + com.hedera.hapi.block.stream.trace.TraceData trace_data = 11; + + /** + * TODO + */ + FilteredSingleItem filtered_single_item = 20; + + /** + * TODO + */ + com.hedera.hapi.block.stream.output.BlockFooter block_footer = 21; } } @@ -252,6 +303,28 @@ enum SubMerkleTree { * of a block, but MAY preclude verification or reconstruction * of consensus state.
*/ +message FilteredItemHash { + /** + * A hash of an item filtered from the stream. + *

+ * The hash algorithm used MUST match the hash algorithm specified in + * the block header for the containing block.
+ * This field is REQUIRED. + */ + bytes item_hash = 1; + + /** + * A record of the merkle path to the item that was filtered + * from the stream.
+ * This path begins at the root of the block proof merkle tree. + *

+ * This REQUIRED field SHALL describe the full path in the virtual + * merkle tree constructed for the block proof that contained the + * item filtered from the stream. + */ + uint64 filtered_path = 3; +} + message FilteredSingleItem { /** * A hash of an item filtered from the stream. From 577d884f190b7051f58f260e317e1b0e224ef5a7 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 6 Oct 2025 22:02:24 -0600 Subject: [PATCH 07/63] spotless Signed-off-by: Matt Hess --- .../app/blocks/impl/BlockStreamBuilder.java | 8 +- .../blocks/impl/BlockStreamManagerImpl.java | 4 +- .../impl/IncrementalStreamingHasher.java | 190 +++++++++--------- .../app/blocks/BlockStreamBuilderTest.java | 46 ++--- .../impl/BlockStreamManagerImplTest.java | 2 +- 5 files changed, 125 insertions(+), 125 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java index 48918db85e20..1e9f8afd69c5 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java @@ -543,7 +543,7 @@ private T toView(@NonNull final BlockItemsTranslator translator, @NonNull fi } List logs = null; for (final var item : blockItems.subList(j, n)) { - // TODO: new trace data implementation + // TODO: new trace data implementation } return (T) switch (view) { @@ -669,7 +669,7 @@ public Output build(final boolean topLevel, @Nullable final List ba builder.logs(logs); } blockItems.add(BlockItem.newBuilder() - // TODO: re-add trace data + // TODO: re-add trace data .build()); } @@ -681,7 +681,7 @@ public Output build(final boolean topLevel, @Nullable final List ba .automaticTokenAssociations( automaticTokenAssociations.getLast().accountId()); blockItems.add(BlockItem.newBuilder() - // TODO: re-add trace data + // TODO: re-add trace data .build()); } // message submit trace data @@ -690,7 +690,7 @@ public Output build(final boolean topLevel, @Nullable final List ba .sequenceNumber(sequenceNumber) .runningHash(runningHash); blockItems.add(BlockItem.newBuilder() - // TODO: re-add trace data + // TODO: re-add trace data .build()); } } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 848b12c5b886..66a4667a63e7 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -769,8 +769,8 @@ protected boolean onExecute() { STATE_CHANGES, ROUND_HEADER, BLOCK_HEADER - // Also EndBlock? - -> { + // Also EndBlock? + -> { MessageDigest digest = sha384DigestOrThrow(); bytes.writeTo(digest); hash = ByteBuffer.wrap(digest.digest()); diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java index 10addc43cb03..e5231952de68 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java @@ -14,107 +14,107 @@ *

This is not thread safe, it is assumed use by single thread.

*/ public class IncrementalStreamingHasher { - /** Prefix byte for hash contents for leaf nodes. */ - private static final byte[] LEAF_PREFIX = new byte[] {0}; - /** Prefix byte for hash contents for internal nodes. */ - private static final byte[] INTERNAL_NODE_PREFIX = new byte[] {2}; - /** The hashing algorithm used for computing the hashes. */ - private final MessageDigest digest; - /** A list to store intermediate hashes as we build the tree. */ - private final LinkedList hashList = new LinkedList<>(); - /** The count of leaves in the tree. */ - private long leafCount = 0; + /** Prefix byte for hash contents for leaf nodes. */ + private static final byte[] LEAF_PREFIX = new byte[] {0}; + /** Prefix byte for hash contents for internal nodes. */ + private static final byte[] INTERNAL_NODE_PREFIX = new byte[] {2}; + /** The hashing algorithm used for computing the hashes. */ + private final MessageDigest digest; + /** A list to store intermediate hashes as we build the tree. */ + private final LinkedList hashList = new LinkedList<>(); + /** The count of leaves in the tree. */ + private long leafCount = 0; - /** Create a new StreamingHasher with an empty state. */ - public IncrementalStreamingHasher() { - try { - digest = MessageDigest.getInstance("SHA-384"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } + /** Create a new StreamingHasher with an empty state. */ + public IncrementalStreamingHasher() { + try { + digest = MessageDigest.getInstance("SHA-384"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } - /** - * Create a StreamingHasher with an existing intermediate hashing state. - * This allows resuming hashing from a previous state. - * - * @param intermediateHashingState the intermediate hashing state - */ - public IncrementalStreamingHasher(List intermediateHashingState) { - this(); - this.hashList.addAll(intermediateHashingState); - } + /** + * Create a StreamingHasher with an existing intermediate hashing state. + * This allows resuming hashing from a previous state. + * + * @param intermediateHashingState the intermediate hashing state + */ + public IncrementalStreamingHasher(List intermediateHashingState) { + this(); + this.hashList.addAll(intermediateHashingState); + } - /** - * Add a new leaf to the Merkle tree. - * - * @param data the data for the new leaf - */ - public void addLeaf(byte[] data) { - final long i = leafCount; - final byte[] e = hashLeaf(data); - hashList.add(e); - for (long n=i; (n & 1L) == 1; n >>= 1) { - final byte[] y = hashList.removeLast(); - final byte[] x = hashList.removeLast(); - hashList.add(hashInternalNode(x, y)); - } - leafCount ++; - } + /** + * Add a new leaf to the Merkle tree. + * + * @param data the data for the new leaf + */ + public void addLeaf(byte[] data) { + final long i = leafCount; + final byte[] e = hashLeaf(data); + hashList.add(e); + for (long n = i; (n & 1L) == 1; n >>= 1) { + final byte[] y = hashList.removeLast(); + final byte[] x = hashList.removeLast(); + hashList.add(hashInternalNode(x, y)); + } + leafCount++; + } - /** - * Compute the Merkle tree root hash from the current state. This does not modify the internal state, so can be - * called at any time and more leaves can be added afterward. - * - * @return the Merkle tree root hash - */ - public byte[] computeRootHash() { - byte[] merkleRootHash = hashList.getLast(); - for (int i = hashList.size() - 2; i >= 0; i--) { - merkleRootHash = hashInternalNode(hashList.get(i), merkleRootHash); - } - return merkleRootHash; - } + /** + * Compute the Merkle tree root hash from the current state. This does not modify the internal state, so can be + * called at any time and more leaves can be added afterward. + * + * @return the Merkle tree root hash + */ + public byte[] computeRootHash() { + byte[] merkleRootHash = hashList.getLast(); + for (int i = hashList.size() - 2; i >= 0; i--) { + merkleRootHash = hashInternalNode(hashList.get(i), merkleRootHash); + } + return merkleRootHash; + } - /** - * Get the current intermediate hashing state. This can be used to save the state and resume hashing later. - * - * @return the intermediate hashing state - */ - public List intermediateHashingState() { - return hashList; - } + /** + * Get the current intermediate hashing state. This can be used to save the state and resume hashing later. + * + * @return the intermediate hashing state + */ + public List intermediateHashingState() { + return hashList; + } - /** - * Get the number of leaves added to the tree so far. - * - * @return the number of leaves - */ - public long leafCount() { - return leafCount; - } + /** + * Get the number of leaves added to the tree so far. + * + * @return the number of leaves + */ + public long leafCount() { + return leafCount; + } - /** - * Hash a leaf node with the appropriate prefix. - * - * @param leafData the data of the leaf - * @return the hash of the leaf node - */ - private byte[] hashLeaf(final byte[] leafData) { - digest.update(LEAF_PREFIX); - return digest.digest(leafData); - } + /** + * Hash a leaf node with the appropriate prefix. + * + * @param leafData the data of the leaf + * @return the hash of the leaf node + */ + private byte[] hashLeaf(final byte[] leafData) { + digest.update(LEAF_PREFIX); + return digest.digest(leafData); + } - /** - * Hash an internal node by combining the hashes of its two children with the appropriate prefix. - * - * @param firstChild the hash of the first child - * @param secondChild the hash of the second child - * @return the hash of the internal node - */ - private byte[] hashInternalNode(final byte[] firstChild, final byte[] secondChild) { - digest.update(INTERNAL_NODE_PREFIX); - digest.update(firstChild); - return digest.digest(secondChild); - } + /** + * Hash an internal node by combining the hashes of its two children with the appropriate prefix. + * + * @param firstChild the hash of the first child + * @param secondChild the hash of the second child + * @return the hash of the internal node + */ + private byte[] hashInternalNode(final byte[] firstChild, final byte[] secondChild) { + digest.update(INTERNAL_NODE_PREFIX); + digest.update(firstChild); + return digest.digest(secondChild); + } } diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java index a7e291cf2523..5485074c0f9d 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java @@ -141,12 +141,12 @@ void testBlockItemsWithTraceAndOutput() { assertTrue(output.hasContractCall()); final var traceItem = blockItems.get(3); - // TODO: assert trace data -// assertTrue(traceItem.hasTraceData()); -// final var trace = traceItem.traceDataOrThrow(); -// assertTrue(trace.hasEvmTraceData()); -// final var evmTrace = trace.evmTraceDataOrThrow(); -// assertEquals(usages, evmTrace.contractSlotUsages()); + // TODO: assert trace data + // assertTrue(traceItem.hasTraceData()); + // final var trace = traceItem.traceDataOrThrow(); + // assertTrue(trace.hasEvmTraceData()); + // final var evmTrace = trace.evmTraceDataOrThrow(); + // assertEquals(usages, evmTrace.contractSlotUsages()); } @Test @@ -162,15 +162,15 @@ void testBlockItemsWithAdditionalAutomaticTokenAssociationTraceData() { final var blockItems = itemsBuilder.build(false, List.of()).blockItems(); final var traceItem = blockItems.get(2); - // TODO: assert trace data -// assertThat(traceItem.hasTraceData()).isTrue(); -// final var trace = traceItem.traceDataOrThrow(); -// -// assertThat(trace.hasAutoAssociateTraceData()).isTrue(); -// final var autoAssociateTraceData = trace.autoAssociateTraceData(); -// assertThat(autoAssociateTraceData).isNotNull(); -// assertThat(autoAssociateTraceData.automaticTokenAssociations().accountNum()) -// .isEqualTo(2); + // TODO: assert trace data + // assertThat(traceItem.hasTraceData()).isTrue(); + // final var trace = traceItem.traceDataOrThrow(); + // + // assertThat(trace.hasAutoAssociateTraceData()).isTrue(); + // final var autoAssociateTraceData = trace.autoAssociateTraceData(); + // assertThat(autoAssociateTraceData).isNotNull(); + // assertThat(autoAssociateTraceData.automaticTokenAssociations().accountNum()) + // .isEqualTo(2); } @Test @@ -181,14 +181,14 @@ void testBlockItemsWithAdditionalSubmitMsgTraceData() { final var blockItems = itemsBuilder.build(false, List.of()).blockItems(); final var traceItem = blockItems.get(2); - // TODO: assert trace data -// assertThat(traceItem.hasTraceData()).isTrue(); -// final var trace = traceItem.traceDataOrThrow(); -// -// assertThat(trace.hasSubmitMessageTraceData()).isTrue(); -// final var submitMessageTraceData = trace.submitMessageTraceData(); -// assertThat(submitMessageTraceData).isNotNull(); -// assertThat(submitMessageTraceData.sequenceNumber()).isEqualTo(66); + // TODO: assert trace data + // assertThat(traceItem.hasTraceData()).isTrue(); + // final var trace = traceItem.traceDataOrThrow(); + // + // assertThat(trace.hasSubmitMessageTraceData()).isTrue(); + // final var submitMessageTraceData = trace.submitMessageTraceData(); + // assertThat(submitMessageTraceData).isNotNull(); + // assertThat(submitMessageTraceData.sequenceNumber()).isEqualTo(66); } @Test diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index d32a8936a96b..250171603e52 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -115,7 +115,7 @@ class BlockStreamManagerImplTest { private static final BlockItem FAKE_STATE_CHANGES = BlockItem.newBuilder() .stateChanges(StateChanges.newBuilder().consensusTimestamp(CONSENSUS_THEN)) .build(); - // TODO: remove, or replace with wrapped record file item + // TODO: remove, or replace with wrapped record file item private static final BlockItem FAKE_RECORD_FILE_ITEM = null; private final InitialStateHash hashInfo = new InitialStateHash(completedFuture(ZERO_BLOCK_HASH), 0); From 43cf3c60d593b967976e07447f700b1a0f8ad227 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 6 Oct 2025 23:16:36 -0600 Subject: [PATCH 08/63] wip Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 16 +++++ .../blocks/impl/BlockStreamManagerImpl.java | 33 +++++++++- .../blocks/impl/IncrementalHasherStorage.java | 64 +++++++++++++++++++ .../impl/IncrementalStreamingHasher.java | 1 + 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index 570caa01cbc9..07a8fd2fca39 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -401,3 +401,19 @@ message RedactedItem { */ SubMerkleTree tree = 3; } + +/** + * TODO + */ +message StreamingTreeSnapshot { + + /** + * Which of the block merkle sub trees this snapshot is for + */ + SubMerkleTree type = 1; + + /** + * All the uncollapsed nodes of the sub tree + */ + repeated bytes nodes = 2; +} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 5cd4f84b8d8b..e6729d5293fb 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -1,6 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.node.app.blocks.impl; +import static com.hedera.hapi.block.stream.SubMerkleTree.CONSENSUS_HEADER; +import static com.hedera.hapi.block.stream.SubMerkleTree.INPUT_ITEM; +import static com.hedera.hapi.block.stream.SubMerkleTree.OUTPUT_ITEM; +import static com.hedera.hapi.block.stream.SubMerkleTree.PREVIOUS_ROOT_HASHES; +import static com.hedera.hapi.block.stream.SubMerkleTree.STATE_CHANGE_ITEM; +import static com.hedera.hapi.block.stream.SubMerkleTree.TRACE_ITEM; import static com.hedera.hapi.node.base.BlockHashAlgorithm.SHA2_384; import static com.hedera.hapi.util.HapiUtils.asInstant; import static com.hedera.hapi.util.HapiUtils.asTimestamp; @@ -24,6 +30,7 @@ import com.hedera.hapi.block.stream.BlockItem; import com.hedera.hapi.block.stream.BlockProof; import com.hedera.hapi.block.stream.MerkleSiblingHash; +import com.hedera.hapi.block.stream.StreamingTreeSnapshot; import com.hedera.hapi.block.stream.SubMerkleTree; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.block.stream.output.StateChanges; @@ -51,6 +58,7 @@ import com.hedera.node.config.types.DiskNetworkExport; import com.hedera.node.internal.network.PendingProof; import com.hedera.pbj.runtime.io.buffer.Bytes; +import com.swirlds.component.framework.schedulers.internal.SequentialTask; import com.swirlds.config.api.Configuration; import com.swirlds.metrics.api.Counter; import com.swirlds.metrics.api.Metrics; @@ -64,9 +72,18 @@ import com.swirlds.state.spi.CommittableWritableStates; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; + +import java.io.BufferedOutputStream; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; import java.security.MessageDigest; import java.time.Duration; import java.time.Instant; @@ -84,6 +101,7 @@ import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; import javax.inject.Inject; @@ -158,7 +176,7 @@ public class BlockStreamManagerImpl implements BlockStreamManager { private final boolean hintsEnabled; private void initIntermediateHashTrees() { - previousBlockHashes = loadHashTree(SubMerkleTree.PREVIOUS_ROOT_HASHES); + previousBlockHashes = loadHashTree(PREVIOUS_ROOT_HASHES); consensusHeaderHasher = loadHashTree(SubMerkleTree.CONSENSUS_HEADER); inputTreeHasher = loadHashTree(SubMerkleTree.INPUT_ITEM); outputTreeHasher = loadHashTree(SubMerkleTree.OUTPUT_ITEM); @@ -179,6 +197,16 @@ static List loadUncollapsedChildren(@NonNull final String basepath, SubM return Collections.emptyList(); } + void writeStreamingSnapshots(@NonNull final String basepath, final long roundNum) { + final var hashingStates = Map.of(PREVIOUS_ROOT_HASHES, previousBlockHashes.intermediateHashingState(), CONSENSUS_HEADER, consensusHeaderHasher.intermediateHashingState(), INPUT_ITEM, inputTreeHasher.intermediateHashingState(), OUTPUT_ITEM, outputTreeHasher.intermediateHashingState(), STATE_CHANGE_ITEM, stateChangesHasher.intermediateHashingState(), TRACE_ITEM, traceDataHasher.intermediateHashingState()); + IncrementalHasherStorage.writeStreamingSnapshots(basepath, hashingStates, roundNum); + } + + Map readStreamingSnapshots(@NonNull final String basepath, final long roundNum) { + //todo + return Map.of(); + } + /** * Represents a block pending completion by the block hash signature needed for its block proof. * @@ -564,7 +592,8 @@ public boolean endRound(@NonNull final State state, final long roundNum) { final var pendingProof = BlockProof.newBuilder() .block(blockNumber) .previousBlockRootHash(lastBlockHash) - .startOfBlockStateRootHash(blockStartStateHash); + .startOfBlockStateRootHash(blockStartStateHash) + .build(); pendingBlocks.add(new PendingBlock( blockNumber, null, diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java new file mode 100644 index 000000000000..dddae5219761 --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.node.app.blocks.impl; + +import com.hedera.hapi.block.stream.StreamingTreeSnapshot; +import com.hedera.hapi.block.stream.SubMerkleTree; +import com.hedera.pbj.runtime.io.buffer.Bytes; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * Storage and reconstruction utility for {@link IncrementalStreamingHasher}. + */ +class IncrementalHasherStorage { + private static final int BUFFER_SIZE = 4 * 1024; // 4KB buffer for optimal I/O + + static StreamingTreeSnapshot readStreamingSnapshot(@NonNull final String filepath) { + // todo + return null; + } + + /** + * todo + * @param basepath + * @param hashingStates + * @param roundNum + */ + static void writeStreamingSnapshots(@NonNull final String basepath, Map> hashingStates, final long roundNum) { + Path created; + try { + created = Files.createDirectory(Path.of(basepath).resolve(String.valueOf(roundNum))); + } catch (IOException e) { + throw new RuntimeException(e); + } + + // write each tree + hashingStates.forEach((type, hasher) -> { + final var snapshot = StreamingTreeSnapshot.newBuilder().type(type).nodes(hasher).build(); + final Path treePath; + try { + treePath = Files.createFile(created.resolve(filenameFor(type))); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + try (BufferedOutputStream out = new BufferedOutputStream(Files.newOutputStream(treePath), BUFFER_SIZE)) { + out.write(StreamingTreeSnapshot.PROTOBUF.toBytes(snapshot).toByteArray()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + static String filenameFor(@NonNull final SubMerkleTree type) { + return type.protoName(); + } +} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java index dffdce8369d0..f332d126928c 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java @@ -83,6 +83,7 @@ public byte[] computeRootHash() { * @return the intermediate hashing state */ public List intermediateHashingState() { + // do we need to copy the arrays here so they don't change? return hashList.stream().map(Bytes::wrap).toList(); } From f69b64c9889ac696c7854ea310dd7f0b5cbc54e5 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 13 Oct 2025 21:47:48 -0600 Subject: [PATCH 09/63] Calculate block hash using v0.68 block merkle tree Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_proof.proto | 30 ++- .../main/java/com/hedera/node/app/Hedera.java | 98 +++++--- .../node/app/blocks/BlockStreamModule.java | 30 +++ .../blocks/impl/BlockStreamManagerImpl.java | 225 +++++++++--------- .../blocks/impl/IncrementalHasherStorage.java | 80 +++---- .../impl/IncrementalStreamingHasher.java | 2 +- .../impl/BlockStreamManagerImplTest.java | 78 +++--- 7 files changed, 313 insertions(+), 230 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto index cdf872f50047..fa82307f4f09 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto @@ -18,6 +18,8 @@ option java_package = "com.hedera.hapi.block.stream.protoc"; // <<>> This comment is special code for setting PBJ Compiler java package option java_multiple_files = true; +import "block/stream/record_file_item.proto"; + /** * A cryptographic proof for the "Block Merkle Tree". * @@ -80,7 +82,7 @@ message BlockProof { * This value SHOULD match the block number of the current block, * under normal operation. */ - uint64 block = 1; + uint64 block = 1 [deprecated = true]; /** * A block root hash for the previous block. @@ -140,7 +142,7 @@ message BlockProof { * of this merkle tree. * */ - bytes previous_block_root_hash = 2; + bytes previous_block_root_hash = 2 [deprecated = true]; /** * A merkle root hash of the network state.
@@ -161,7 +163,7 @@ message BlockProof { * Stateless (non-state-processing) clients MUST use this value to * construct the block merkle tree. */ - bytes start_of_block_state_root_hash = 3; + bytes start_of_block_state_root_hash = 3 [deprecated = true]; /** * A TSS signature for one block.
@@ -182,7 +184,7 @@ message BlockProof { * threshold was met if the signature itself can be validated with * the network public key (a.k.a `LedgerID`). */ - bytes block_signature = 4; + bytes block_signature = 4 [deprecated = true]; /** * A set of hash values along with ordering information.
@@ -211,7 +213,7 @@ message BlockProof { * "secondary" root hash MUST then be verified using * the value of `block_signature`. */ - repeated MerkleSiblingHash sibling_hashes = 5; + repeated MerkleSiblingHash sibling_hashes = 5 [deprecated = true]; /** * The hinTS key that this signature verifies under; a stream consumer should @@ -224,6 +226,24 @@ message BlockProof { * from the network's ledger id. */ ChainOfTrustProof verification_key_proof = 7; + + oneof proof { + TssSignedBlockProof signed_block_proof = 8; +// StateProof block_state_proof = 9; + SignedRecordFileProof signed_record_file_proof = 10; + } +} + +message TssSignedBlockProof { + bytes block_signature = 1; +} + +message SignedRecordFileProof { + /** + * A collection of RSA signatures from consensus nodes.
+ * These signatures validate the hash of the record_file_contents field. + */ + repeated RecordFileSignature record_file_signatures = 1; } /** diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java index 3555d9d34a98..d20773500dd8 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java @@ -9,11 +9,7 @@ import static com.hedera.hapi.util.HapiUtils.SEMANTIC_VERSION_COMPARATOR; import static com.hedera.hapi.util.HapiUtils.functionOf; import static com.hedera.node.app.blocks.BlockStreamManager.ZERO_BLOCK_HASH; -import static com.hedera.node.app.blocks.impl.BlockImplUtils.combine; -import static com.hedera.node.app.blocks.impl.BlockStreamManagerImpl.NULL_HASH; -import static com.hedera.node.app.blocks.impl.ConcurrentStreamingTreeHasher.rootHashFrom; import static com.hedera.node.app.blocks.schemas.V0560BlockStreamSchema.BLOCK_STREAM_INFO_STATE_ID; -import static com.hedera.node.app.hapi.utils.CommonUtils.noThrowSha384HashOf; import static com.hedera.node.app.records.impl.BlockRecordInfoUtils.blockHashByBlockNumber; import static com.hedera.node.app.records.schemas.V0490BlockRecordSchema.BLOCKS_STATE_ID; import static com.hedera.node.app.spi.workflows.record.StreamBuilder.nodeSignedTxWith; @@ -30,7 +26,6 @@ import static org.hiero.consensus.model.status.PlatformStatus.ACTIVE; import static org.hiero.consensus.model.status.PlatformStatus.STARTING_UP; -import com.hedera.hapi.block.stream.BlockItem; import com.hedera.hapi.block.stream.output.SingletonUpdateChange; import com.hedera.hapi.block.stream.output.StateChange; import com.hedera.hapi.block.stream.output.StateChanges; @@ -54,10 +49,10 @@ import com.hedera.node.app.blocks.BlockStreamManager; import com.hedera.node.app.blocks.BlockStreamService; import com.hedera.node.app.blocks.InitialStateHash; -import com.hedera.node.app.blocks.StreamingTreeHasher; import com.hedera.node.app.blocks.impl.BlockStreamManagerImpl; import com.hedera.node.app.blocks.impl.BoundaryStateChangeListener; import com.hedera.node.app.blocks.impl.ImmediateStateChangeListener; +import com.hedera.node.app.blocks.impl.IncrementalStreamingHasher; import com.hedera.node.app.config.BootstrapConfigProviderImpl; import com.hedera.node.app.config.ConfigProviderImpl; import com.hedera.node.app.fees.FeeService; @@ -1273,8 +1268,9 @@ private Bytes startBlockHashFrom(@NonNull final State state) { .getSingleton(BLOCK_STREAM_INFO_STATE_ID) .get(); requireNonNull(blockStreamInfo); - // Three of the four ingredients in the block hash are directly in the BlockStreamInfo; that is, - // the previous block hash, the input tree root hash, and the start of block state hash + + // Most of the ingredients in the block hash are directly in the BlockStreamInfo + // Branch 1: lastBlockHash final var prevBlockHash = blockStreamInfo.blockNumber() == 0L ? ZERO_BLOCK_HASH : blockHashByBlockNumber( @@ -1282,19 +1278,56 @@ private Bytes startBlockHashFrom(@NonNull final State state) { blockStreamInfo.blockNumber() - 1, blockStreamInfo.blockNumber() - 1); requireNonNull(prevBlockHash); - - // The fourth ingredient, the state changes tree root hash, is not directly in the BlockStreamInfo, but - // we can recompute it based on the tree hash information and the fact the last state changes item in - // the block was devoted to putting the BlockStreamInfo itself into the state - final var stateChangesHash = stateChangesTreeRootHashFrom(blockStreamInfo); - - final var level1A = combine(prevBlockHash, blockStreamInfo.startOfBlockStateHash()); - final var level1B = combine(blockStreamInfo.consensusHeaderTreeRootHash(), blockStreamInfo.inputTreeRootHash()); - final var level1C = combine(blockStreamInfo.outputTreeRootHash(), stateChangesHash); - final var level1D = combine(blockStreamInfo.traceDataTreeRootHash(), NULL_HASH); - final var leftParent = combine(level1A, level1B); - final var rightParent = combine(level1C, level1D); - return combine(leftParent, rightParent); + // Branch 2 + final var prevBlockRootsHash = + Bytes.wrap(new IncrementalStreamingHasher(blockStreamInfo.intermediatePreviousBlockRootHashes().stream() + .map(Bytes::toByteArray) + .toList()) + .computeRootHash()); + // Branch 3: + final var blockStartStateHash = blockStreamInfo.startOfBlockStateHash(); + // Branch 4 + final var consensusHeaderHash = + Bytes.wrap(new IncrementalStreamingHasher(blockStreamInfo.intermediatePreviousBlockRootHashes().stream() + .map(Bytes::toByteArray) + .toList()) + .computeRootHash()); + // Branch 5 + final var inputsHash = + Bytes.wrap(new IncrementalStreamingHasher(blockStreamInfo.intermediatePreviousBlockRootHashes().stream() + .map(Bytes::toByteArray) + .toList()) + .computeRootHash()); + // Branch 6 + final var outputsHash = + Bytes.wrap(new IncrementalStreamingHasher(blockStreamInfo.intermediatePreviousBlockRootHashes().stream() + .map(Bytes::toByteArray) + .toList()) + .computeRootHash()); + // Branch 7, the state changes hash, will come immediately following + // Branch 8 + final var traceDataHash = + Bytes.wrap(new IncrementalStreamingHasher(blockStreamInfo.intermediatePreviousBlockRootHashes().stream() + .map(Bytes::toByteArray) + .toList()) + .computeRootHash()); + + // The final ingredient, the state changes tree root hash (branch 7), is not directly in the BlockStreamInfo, + // but we can recompute it based on the tree hash information and the fact the last state changes item in the + // block was devoted to putting the BlockStreamInfo itself into the state + final var stateChangesHash = Bytes.wrap(stateChangesSubTreeRootHashFrom(blockStreamInfo)); + + return BlockStreamManagerImpl.combine( + prevBlockHash, + prevBlockRootsHash, + blockStartStateHash, + consensusHeaderHash, + inputsHash, + outputsHash, + stateChangesHash, + traceDataHash, + // TODO: use the correct timestamp + blockStreamInfo.lastIntervalProcessTime()); } /** @@ -1305,24 +1338,23 @@ private Bytes startBlockHashFrom(@NonNull final State state) { * @param info the context to use * @return the inferred output tree root hash */ - private @NonNull Bytes stateChangesTreeRootHashFrom(@NonNull final BlockStreamInfo info) { - // This was the last state change in the block + private @NonNull byte[] stateChangesSubTreeRootHashFrom(@NonNull final BlockStreamInfo info) { + // Construct the final state change final var blockStreamInfoChange = StateChange.newBuilder() .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal()) .singletonUpdate(SingletonUpdateChange.newBuilder() .blockStreamInfoValue(info) .build()) .build(); - // And this was the last output block item - final var lastStateChanges = BlockItem.newBuilder() - .stateChanges(new StateChanges(info.blockEndTime(), List.of(blockStreamInfoChange))) - .build(); - // So we can combine this last leaf's has with the size and rightmost hashes - // store from the pending state changes tree to recompute its final root hash - final var penultimateStateChangesTreeStatus = new StreamingTreeHasher.Status( - info.numPrecedingStateChangesItems(), info.rightmostPrecedingStateChangesTreeHashes()); - final var lastLeafHash = noThrowSha384HashOf(BlockItem.PROTOBUF.toBytes(lastStateChanges)); - return rootHashFrom(penultimateStateChangesTreeStatus, lastLeafHash); + final var changeBytes = StateChange.PROTOBUF.toBytes(blockStreamInfoChange); + + // Add the final state change as a leaf and compute the root + final var stateChangeSubTree = + new IncrementalStreamingHasher(info.intermediateStateChangeBlockItemHashes().stream() + .map(Bytes::toByteArray) + .toList()); + stateChangeSubTree.addLeaf(changeBytes.toByteArray()); + return stateChangeSubTree.computeRootHash(); } private void logConfiguration() { diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamModule.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamModule.java index 28c3b3ecea97..88c0927830db 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamModule.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamModule.java @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.node.app.blocks; +import com.google.protobuf.ByteString; import com.hedera.node.app.blocks.impl.BlockStreamManagerImpl; import com.hedera.node.app.blocks.impl.BoundaryStateChangeListener; import com.hedera.node.app.blocks.impl.streaming.BlockBufferService; @@ -8,11 +9,13 @@ import com.hedera.node.app.blocks.impl.streaming.FileAndGrpcBlockItemWriter; import com.hedera.node.app.blocks.impl.streaming.FileBlockItemWriter; import com.hedera.node.app.blocks.impl.streaming.GrpcBlockItemWriter; +import com.hedera.node.app.blocks.schemas.V0560BlockStreamSchema; import com.hedera.node.app.metrics.BlockStreamMetrics; import com.hedera.node.app.services.NodeRewardManager; import com.hedera.node.app.spi.info.NodeInfo; import com.hedera.node.config.ConfigProvider; import com.hedera.node.config.data.BlockStreamConfig; +import com.hederahashgraph.api.proto.java.BlockStreamInfo; import com.swirlds.metrics.api.Metrics; import com.swirlds.state.State; import dagger.Module; @@ -46,6 +49,33 @@ static BlockNodeConnectionManager provideBlockNodeConnectionManager( return manager; } + @Provides + @Singleton + static BlockStreamManagerImpl.StateHashes provideStateHashes(@NonNull final State state) { + var blockStreamInfo = state.getReadableStates(BlockStreamService.NAME) + .getSingleton(V0560BlockStreamSchema.BLOCK_STREAM_INFO_STATE_ID) + .get(); + return new BlockStreamManagerImpl.StateHashes( + blockStreamInfo.getIntermediatePreviousBlockRootHashesList().stream() + .map(ByteString::toByteArray) + .toList(), + blockStreamInfo.getIntermediateConsensusHeaderHashesList().stream() + .map(ByteString::toByteArray) + .toList(), + blockStreamInfo.getIntermediateInputBlockItemHashesList().stream() + .map(ByteString::toByteArray) + .toList(), + blockStreamInfo.getIntermediateOutputBlockItemHashesList().stream() + .map(ByteString::toByteArray) + .toList(), + blockStreamInfo.getIntermediateStateChangeBlockItemHashesList().stream() + .map(ByteString::toByteArray) + .toList(), + blockStreamInfo.getIntermediateTraceDataHashesList().stream() + .map(ByteString::toByteArray) + .toList()); + } + @Provides @Singleton static BlockStreamMetrics provideBlockStreamMetrics(@NonNull final Metrics metrics) { diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index b28a464b40b8..d4912a2aed1c 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -1,12 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.node.app.blocks.impl; -import static com.hedera.hapi.block.stream.SubMerkleTree.CONSENSUS_HEADER; -import static com.hedera.hapi.block.stream.SubMerkleTree.INPUT_ITEM; -import static com.hedera.hapi.block.stream.SubMerkleTree.OUTPUT_ITEM; -import static com.hedera.hapi.block.stream.SubMerkleTree.PREVIOUS_ROOT_HASHES; -import static com.hedera.hapi.block.stream.SubMerkleTree.STATE_CHANGE_ITEM; -import static com.hedera.hapi.block.stream.SubMerkleTree.TRACE_ITEM; import static com.hedera.hapi.node.base.BlockHashAlgorithm.SHA2_384; import static com.hedera.hapi.util.HapiUtils.asInstant; import static com.hedera.hapi.util.HapiUtils.asTimestamp; @@ -31,8 +25,6 @@ import com.hedera.hapi.block.stream.BlockProof; import com.hedera.hapi.block.stream.ChainOfTrustProof; import com.hedera.hapi.block.stream.MerkleSiblingHash; -import com.hedera.hapi.block.stream.StreamingTreeSnapshot; -import com.hedera.hapi.block.stream.SubMerkleTree; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.block.stream.output.StateChanges; import com.hedera.hapi.node.base.SemanticVersion; @@ -59,7 +51,6 @@ import com.hedera.node.config.types.DiskNetworkExport; import com.hedera.node.internal.network.PendingProof; import com.hedera.pbj.runtime.io.buffer.Bytes; -import com.swirlds.component.framework.schedulers.internal.SequentialTask; import com.swirlds.config.api.Configuration; import com.swirlds.metrics.api.Counter; import com.swirlds.metrics.api.Metrics; @@ -73,23 +64,13 @@ import com.swirlds.state.spi.CommittableWritableStates; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; - -import java.io.BufferedOutputStream; -import java.io.BufferedWriter; -import java.io.FileWriter; -import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.ByteBuffer; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.FileAttribute; import java.security.MessageDigest; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -102,7 +83,6 @@ import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; import javax.inject.Inject; @@ -160,9 +140,6 @@ public class BlockStreamManagerImpl implements BlockStreamManager { private BlockItemWriter writer; private Instant firstConsensusTimeOfCurrentBlock; - // TODO: set from config, e.g. provider.getConfiguration().getValue("blocktree.uncollapsedHashesDir", String.class); - private String intermediateHashesDir = "data/intStreams"; - // block merkle tree private IncrementalStreamingHasher previousBlockHashes; // ALL previous hashes, but streaming-collapsed private Bytes stateHashAtStartOfBlock; @@ -176,38 +153,6 @@ public class BlockStreamManagerImpl implements BlockStreamManager { private BlockStreamManagerTask worker; private final boolean hintsEnabled; - private void initIntermediateHashTrees() { - previousBlockHashes = loadHashTree(PREVIOUS_ROOT_HASHES); - consensusHeaderHasher = loadHashTree(SubMerkleTree.CONSENSUS_HEADER); - inputTreeHasher = loadHashTree(SubMerkleTree.INPUT_ITEM); - outputTreeHasher = loadHashTree(SubMerkleTree.OUTPUT_ITEM); - stateChangesHasher = loadHashTree(SubMerkleTree.STATE_CHANGE_ITEM); - traceDataHasher = loadHashTree(SubMerkleTree.TRACE_ITEM); - } - - private IncrementalStreamingHasher loadHashTree(final SubMerkleTree subtreeType) { - final var iHashes = loadUncollapsedChildren(intermediateHashesDir, subtreeType); - return new IncrementalStreamingHasher(iHashes); - } - - static List loadUncollapsedChildren(@NonNull final String basepath, SubMerkleTree subtreeType) { - final var filepath = - Path.of(basepath).resolve(subtreeType.protoName() + ""); // .smt for subMerkleTree ? - - // TODO load file bytes here - return Collections.emptyList(); - } - - void writeStreamingSnapshots(@NonNull final String basepath, final long roundNum) { - final var hashingStates = Map.of(PREVIOUS_ROOT_HASHES, previousBlockHashes.intermediateHashingState(), CONSENSUS_HEADER, consensusHeaderHasher.intermediateHashingState(), INPUT_ITEM, inputTreeHasher.intermediateHashingState(), OUTPUT_ITEM, outputTreeHasher.intermediateHashingState(), STATE_CHANGE_ITEM, stateChangesHasher.intermediateHashingState(), TRACE_ITEM, traceDataHasher.intermediateHashingState()); - IncrementalHasherStorage.writeStreamingSnapshots(basepath, hashingStates, roundNum); - } - - Map readStreamingSnapshots(@NonNull final String basepath, final long roundNum) { - //todo - return Map.of(); - } - /** * Represents a block pending completion by the block hash signature needed for its block proof. * @@ -268,6 +213,14 @@ public void flushPending(final boolean withSiblingHashes) { */ private final Counter indirectProofCounter; + public record StateHashes( + @NonNull List previousBlockRootHashes, + @NonNull List consensusHeaderHashes, + @NonNull List inputItemHashes, + @NonNull List outputItemHashes, + @NonNull List stateChangeHashes, + @NonNull List traceDataHashes) {} + @Inject public BlockStreamManagerImpl( @NonNull final BlockHashSigner blockHashSigner, @@ -280,7 +233,8 @@ public BlockStreamManagerImpl( @NonNull final SemanticVersion version, @NonNull final PlatformStateFacade platformStateFacade, @NonNull final Lifecycle lifecycle, - @NonNull final Metrics metrics) { + @NonNull final Metrics metrics, + @NonNull final StateHashes intermediatHashes) { this.blockHashSigner = requireNonNull(blockHashSigner); this.networkInfo = requireNonNull(networkInfo); this.version = requireNonNull(version); @@ -309,7 +263,14 @@ public BlockStreamManagerImpl( indirectProofCounter = requireNonNull(metrics) .getOrCreate(new Counter.Config("block", "numIndirectProofs") .withDescription("Number of blocks closed with indirect proofs")); - initIntermediateHashTrees(); + + previousBlockHashes = new IncrementalStreamingHasher(intermediatHashes.previousBlockRootHashes); + consensusHeaderHasher = new IncrementalStreamingHasher(intermediatHashes.consensusHeaderHashes); + inputTreeHasher = new IncrementalStreamingHasher(intermediatHashes.inputItemHashes); + outputTreeHasher = new IncrementalStreamingHasher(intermediatHashes.outputItemHashes); + stateChangesHasher = new IncrementalStreamingHasher(intermediatHashes.stateChangeHashes); + traceDataHasher = new IncrementalStreamingHasher(intermediatHashes.traceDataHashes); + log.info( "Initialized BlockStreamManager from round {} with end-of-round hash {}", lastRoundOfPrevBlock, @@ -467,6 +428,72 @@ public void setLastTopLevelTime(@NonNull final Instant lastTopLevelTime) { return asInstant(lastUsedTime); } + public static Bytes combine( + final Bytes prevBlockHash, + final Bytes prevBlockRootsHash, + final Bytes startingStateHash, + final Bytes consensusHeaderHash, + final Bytes inputsHash, + final Bytes outputsHash, + final Bytes stateChangesHash, + final Bytes traceDataHash, + final Timestamp firstConsensusTimeOfCurrentBlock) { + return combine( + prevBlockHash, + prevBlockRootsHash, + startingStateHash, + consensusHeaderHash, + inputsHash, + outputsHash, + stateChangesHash, + traceDataHash, + asInstant(firstConsensusTimeOfCurrentBlock)); + } + + public static Bytes combine( + final Bytes prevBlockHash, + final Bytes prevBlockRootsHash, + final Bytes startingStateHash, + final Bytes consensusHeaderHash, + final Bytes inputsHash, + final Bytes outputsHash, + final Bytes stateChangesHash, + final Bytes traceDataHash, + final Instant firstConsensusTimeOfCurrentBlock) { + // Compute depth four hashes + final var depth4Node1 = BlockImplUtils.combine(prevBlockHash, prevBlockRootsHash); + final var depth4Node2 = BlockImplUtils.combine(startingStateHash, consensusHeaderHash); + final var depth4Node3 = BlockImplUtils.combine(inputsHash, outputsHash); + final var depth4Node4 = BlockImplUtils.combine(stateChangesHash, traceDataHash); + + final var combinedNulls = BlockImplUtils.combine(NULL_HASH, NULL_HASH); + final var depth4Node5 = combinedNulls; + final var depth4Node6 = combinedNulls; + final var depth4Node7 = combinedNulls; + final var depth4Node8 = combinedNulls; + + // Compute depth three hashes + final var depth3Node1 = BlockImplUtils.combine(depth4Node1, depth4Node2); + final var depth3Node2 = BlockImplUtils.combine(depth4Node3, depth4Node4); + final var depth3Node3 = BlockImplUtils.combine(depth4Node5, depth4Node6); + final var depth3Node4 = BlockImplUtils.combine(depth4Node7, depth4Node8); + + // Compute depth two hashes + final var depth2Node1 = BlockImplUtils.combine(depth3Node1, depth3Node2); + final var depth2Node2 = BlockImplUtils.combine(depth3Node3, depth3Node4); + + // Compute depth one hashes + final var timestamp = Timestamp.PROTOBUF.toBytes(Timestamp.newBuilder() + .seconds(firstConsensusTimeOfCurrentBlock.getEpochSecond()) + .nanos(firstConsensusTimeOfCurrentBlock.getNano()) + .build()); + final var depth1Node0 = noThrowSha384HashOf(timestamp); + final var depth1Node1 = BlockImplUtils.combine(depth2Node1, depth2Node2); + + // Compute the block's root hash + return BlockImplUtils.combine(depth1Node0, depth1Node1); + } + @Override public boolean endRound(@NonNull final State state, final long roundNum) { final var storeFactory = new ReadableStoreFactory(state); @@ -480,8 +507,8 @@ public boolean endRound(@NonNull final State state, final long roundNum) { if (state instanceof VirtualMapState hederaNewStateRoot) { hederaNewStateRoot.commitSingletons(); } - // Flush all boundary state changes besides the BlockStreamInfo + // Flush all boundary state changes besides the BlockStreamInfo worker.addItem(flushChangesFromListener(boundaryStateChangeListener)); worker.sync(); @@ -534,38 +561,28 @@ public boolean endRound(@NonNull final State state, final long roundNum) { traceDataHasher.intermediateHashingState())); ((CommittableWritableStates) writableState).commit(); + // Produce one more state change item (i.e. putting the block stream info just constructed into state) worker.addItem(flushChangesFromListener(boundaryStateChangeListener)); worker.sync(); - // Compute depth four hashes - final var depth4Node1 = combine(lastBlockHash, prevBlockRootsHash); - final var depth4Node2 = combine(blockStartStateHash, consensusHeaderHash); - final var depth4Node3 = combine(inputsHash, outputsHash); - final var depth4Node4 = combine(stateChangesHash, traceDataHash); - - final var combinedNulls = combine(NULL_HASH, NULL_HASH); - final var depth4Node5 = combinedNulls; - final var depth4Node6 = combinedNulls; - final var depth4Node7 = combinedNulls; - final var depth4Node8 = combinedNulls; - - // Compute depth three hashes - final var depth3Node1 = combine(depth4Node1, depth4Node2); - final var depth3Node2 = combine(depth4Node3, depth4Node4); - final var depth3Node3 = combine(depth4Node5, depth4Node6); - final var depth3Node4 = combine(depth4Node7, depth4Node8); - - // TODO(#21210): Implement streaming merkle tree of all block hashes from genesis to N-1 - // For now, using NULL_HASH as placeholder until the historical block data infrastructure is ready. - final var blockHashesTreeRoot = NULL_HASH; + final var blockHash = combine( + lastBlockHash, + prevBlockRootsHash, + stateHashAtStartOfBlock, + consensusHeaderHash, + inputsHash, + outputsHash, + stateChangesHash, + traceDataHash, + firstConsensusTimeOfCurrentBlock); // Create BlockFooter with the three essential hashes: // 1. previousBlockRootHash - Root hash of the previous block (N-1) - // 2. rootHashOfAllBlockHashesTree - Streaming tree of all block hashes 0..N-1 (TODO: #21210) + // 2. rootHashOfAllBlockHashesTree - Streaming tree of all block hashes 0..N-1 // 3. startOfBlockStateRootHash - State hash at the beginning of current block final var blockFooter = com.hedera.hapi.block.stream.output.BlockFooter.newBuilder() .previousBlockRootHash(lastBlockHash) - .rootHashOfAllBlockHashesTree(blockHashesTreeRoot) + .rootHashOfAllBlockHashesTree(blockHash) .startOfBlockStateRootHash(blockStartStateHash) .build(); @@ -575,35 +592,17 @@ public boolean endRound(@NonNull final State state, final long roundNum) { worker.addItem(footerItem); worker.sync(); - // Compute depth two hashes - final var depth2Node1 = combine(depth3Node1, depth3Node2); - final var depth2Node2 = combine(depth3Node3, depth3Node4); - - // Compute depth one hashes - final var timestamp = Timestamp.PROTOBUF.toBytes(Timestamp.newBuilder() - .seconds(firstConsensusTimeOfCurrentBlock.getEpochSecond()) - .nanos(firstConsensusTimeOfCurrentBlock.getNano()) - .build()); - final var depth1Node0 = noThrowSha384HashOf(timestamp); - final var depth1Node1 = combine(depth2Node1, depth2Node2); - - // Compute the block's root hash - final var blockHash = combine(depth1Node0, depth1Node1); - - final var pendingProof = BlockProof.newBuilder() - .block(blockNumber) - .previousBlockRootHash(lastBlockHash) - .startOfBlockStateRootHash(blockStartStateHash) - .build(); - pendingBlocks.add(new PendingBlock( - blockNumber, - null, - blockHash, - pendingProof, - writer, - new MerkleSiblingHash(false, blockStartStateHash), - new MerkleSiblingHash(false, depth2Node1), - new MerkleSiblingHash(false, depth1Node1))); + // TODO: construct pending block proof + final var pendingProof = BlockProof.newBuilder(); + // pendingBlocks.add(new PendingBlock( + // blockNumber, + // null, + // blockHash, + // pendingProof, + // writer, + // new MerkleSiblingHash(false, blockStartStateHash), + // new MerkleSiblingHash(false, depth2Node1), + // new MerkleSiblingHash(false, depth1Node1))); // Update in-memory state to prepare for the next block lastBlockHash = blockHash; @@ -651,7 +650,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { } requireNonNull(fatalShutdownFuture).complete(null); - // TODO: write intermediate hashes of sub trees + // TODO: write intermediate hashes of sub trees? } return closesBlock; } @@ -905,8 +904,8 @@ protected boolean onExecute() { hash.rewind(); outputTreeHasher.addLeaf(hash.array()); } - case TRANSACTION_OUTPUT, BLOCK_HEADER -> outputTreeHasher.addLeaf(hash); - case STATE_CHANGES -> stateChangesHasher.addLeaf(hash); + case TRANSACTION_OUTPUT, BLOCK_HEADER -> outputTreeHasher.addLeaf(hash.array()); + case STATE_CHANGES -> stateChangesHasher.addLeaf(hash.array()); case BLOCK_FOOTER, BLOCK_PROOF -> { // BlockFooter and BlockProof are not included in any merkle tree // They are metadata about the block, not part of the hashed content diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java index dddae5219761..72c99611755e 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java @@ -4,7 +4,7 @@ import com.hedera.hapi.block.stream.StreamingTreeSnapshot; import com.hedera.hapi.block.stream.SubMerkleTree; import com.hedera.pbj.runtime.io.buffer.Bytes; - +import edu.umd.cs.findbugs.annotations.NonNull; import java.io.BufferedOutputStream; import java.io.IOException; import java.io.UncheckedIOException; @@ -13,52 +13,52 @@ import java.util.List; import java.util.Map; -import edu.umd.cs.findbugs.annotations.NonNull; - /** * Storage and reconstruction utility for {@link IncrementalStreamingHasher}. */ class IncrementalHasherStorage { - private static final int BUFFER_SIZE = 4 * 1024; // 4KB buffer for optimal I/O + private static final int BUFFER_SIZE = 4 * 1024; // 4KB - static StreamingTreeSnapshot readStreamingSnapshot(@NonNull final String filepath) { - // todo - return null; - } + static StreamingTreeSnapshot readStreamingSnapshot(@NonNull final String filepath) { + // todo + return null; + } - /** - * todo - * @param basepath - * @param hashingStates - * @param roundNum - */ - static void writeStreamingSnapshots(@NonNull final String basepath, Map> hashingStates, final long roundNum) { - Path created; - try { - created = Files.createDirectory(Path.of(basepath).resolve(String.valueOf(roundNum))); - } catch (IOException e) { - throw new RuntimeException(e); - } + /** + * todo + * @param basepath + * @param hashingStates + * @param roundNum + */ + static void writeStreamingSnapshots( + @NonNull final String basepath, Map> hashingStates, final long roundNum) { + Path created; + try { + created = Files.createDirectory(Path.of(basepath).resolve(String.valueOf(roundNum))); + } catch (IOException e) { + throw new RuntimeException(e); + } - // write each tree - hashingStates.forEach((type, hasher) -> { - final var snapshot = StreamingTreeSnapshot.newBuilder().type(type).nodes(hasher).build(); - final Path treePath; - try { - treePath = Files.createFile(created.resolve(filenameFor(type))); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + // write each tree + hashingStates.forEach((type, hasher) -> { + final var snapshot = + StreamingTreeSnapshot.newBuilder().type(type).nodes(hasher).build(); + final Path treePath; + try { + treePath = Files.createFile(created.resolve(filenameFor(type))); + } catch (IOException e) { + throw new UncheckedIOException(e); + } - try (BufferedOutputStream out = new BufferedOutputStream(Files.newOutputStream(treePath), BUFFER_SIZE)) { - out.write(StreamingTreeSnapshot.PROTOBUF.toBytes(snapshot).toByteArray()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - } + try (BufferedOutputStream out = new BufferedOutputStream(Files.newOutputStream(treePath), BUFFER_SIZE)) { + out.write(StreamingTreeSnapshot.PROTOBUF.toBytes(snapshot).toByteArray()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } - static String filenameFor(@NonNull final SubMerkleTree type) { - return type.protoName(); - } + static String filenameFor(@NonNull final SubMerkleTree type) { + return type.protoName(); + } } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java index f332d126928c..0c8f8a245252 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java @@ -83,7 +83,7 @@ public byte[] computeRootHash() { * @return the intermediate hashing state */ public List intermediateHashingState() { - // do we need to copy the arrays here so they don't change? + // do we need to copy the arrays here so they don't change? return hashList.stream().map(Bytes::wrap).toList(); } diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index 496522f0589f..7f533c6682be 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -13,7 +13,6 @@ import static com.hedera.node.app.hapi.utils.CommonUtils.noThrowSha384HashOf; import static com.swirlds.platform.state.service.schemas.V0540PlatformStateSchema.PLATFORM_STATE_STATE_ID; import static com.swirlds.platform.state.service.schemas.V0540PlatformStateSchema.PLATFORM_STATE_STATE_LABEL; -import static com.swirlds.platform.test.fixtures.state.TestPlatformStateFacade.TEST_PLATFORM_STATE_FACADE; import static java.util.concurrent.CompletableFuture.completedFuture; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -72,9 +71,9 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -234,18 +233,19 @@ void classifiesNonGenesisBlockOfSameVersionWithWorkDoneAsNoWork() { @Test void canUpdateDistinguishedTimes() { given(configProvider.getConfiguration()).willReturn(new VersionedConfigImpl(DEFAULT_CONFIG, 1L)); - subject = new BlockStreamManagerImpl( - blockHashSigner, - () -> aWriter, - ForkJoinPool.commonPool(), - configProvider, - networkInfo, - boundaryStateChangeListener, - hashInfo, - SemanticVersion.DEFAULT, - TEST_PLATFORM_STATE_FACADE, - lifecycle, - metrics); + // TODO: fix + // subject = new BlockStreamManagerImpl( + // blockHashSigner, + // () -> aWriter, + // ForkJoinPool.commonPool(), + // configProvider, + // networkInfo, + // boundaryStateChangeListener, + // hashInfo, + // SemanticVersion.DEFAULT, + // TEST_PLATFORM_STATE_FACADE, + // lifecycle, + // metrics); assertSame(Instant.EPOCH, subject.lastIntervalProcessTime()); subject.setLastIntervalProcessTime(CONSENSUS_NOW); assertEquals(CONSENSUS_NOW, subject.lastIntervalProcessTime()); @@ -258,18 +258,19 @@ void canUpdateDistinguishedTimes() { @Test void requiresLastHashToBeInitialized() { given(configProvider.getConfiguration()).willReturn(new VersionedConfigImpl(DEFAULT_CONFIG, 1)); - subject = new BlockStreamManagerImpl( - blockHashSigner, - () -> aWriter, - ForkJoinPool.commonPool(), - configProvider, - networkInfo, - boundaryStateChangeListener, - hashInfo, - SemanticVersion.DEFAULT, - TEST_PLATFORM_STATE_FACADE, - lifecycle, - metrics); + // TODO: fix + // subject = new BlockStreamManagerImpl( + // blockHashSigner, + // () -> aWriter, + // ForkJoinPool.commonPool(), + // configProvider, + // networkInfo, + // boundaryStateChangeListener, + // hashInfo, + // SemanticVersion.DEFAULT, + // TEST_PLATFORM_STATE_FACADE, + // lifecycle, + // metrics); assertThrows(IllegalStateException.class, () -> subject.startRound(round, state)); } @@ -1184,18 +1185,19 @@ private void givenSubjectWith( .withValue("blockStream.blockPeriod", Duration.of(blockPeriod, ChronoUnit.SECONDS)) .getOrCreateConfig(); given(configProvider.getConfiguration()).willReturn(new VersionedConfigImpl(config, 1L)); - subject = new BlockStreamManagerImpl( - blockHashSigner, - () -> writers[nextWriter.getAndIncrement()], - ForkJoinPool.commonPool(), - configProvider, - networkInfo, - boundaryStateChangeListener, - hashInfo, - SemanticVersion.DEFAULT, - TEST_PLATFORM_STATE_FACADE, - lifecycle, - metrics); + // TODO: fix + // subject = new BlockStreamManagerImpl( + // blockHashSigner, + // () -> writers[nextWriter.getAndIncrement()], + // ForkJoinPool.commonPool(), + // configProvider, + // networkInfo, + // boundaryStateChangeListener, + // hashInfo, + // SemanticVersion.DEFAULT, + // TEST_PLATFORM_STATE_FACADE, + // lifecycle, + // metrics); given(state.getReadableStates(any())).willReturn(readableStates); given(readableStates.getSingleton(PLATFORM_STATE_STATE_ID)).willReturn(platformStateReadableSingletonState); lenient().when(state.getReadableStates(FreezeServiceImpl.NAME)).thenReturn(readableStates); From 916f7d0658960c9fdae8de83e341ff148f44c5f8 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Tue, 14 Oct 2025 15:24:15 -0600 Subject: [PATCH 10/63] Initialize block trees and last block hash Signed-off-by: Matt Hess --- .../main/java/com/hedera/node/app/Hedera.java | 122 +--------- .../node/app/blocks/BlockStreamManager.java | 15 +- .../node/app/blocks/BlockStreamModule.java | 30 --- .../blocks/impl/BlockStreamManagerImpl.java | 125 +++++++++- .../impl/BlockStreamManagerImplTest.java | 223 +++++++++--------- .../schemas/V0560BlockStreamSchemaTest.java | 38 +-- 6 files changed, 261 insertions(+), 292 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java index d20773500dd8..203a11315f4c 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.node.app; -import static com.hedera.hapi.block.stream.output.StateIdentifier.STATE_ID_BLOCK_STREAM_INFO; import static com.hedera.hapi.node.base.ResponseCodeEnum.DUPLICATE_TRANSACTION; import static com.hedera.hapi.node.base.ResponseCodeEnum.NOT_SUPPORTED; import static com.hedera.hapi.node.base.ResponseCodeEnum.PLATFORM_NOT_ACTIVE; @@ -9,8 +8,6 @@ import static com.hedera.hapi.util.HapiUtils.SEMANTIC_VERSION_COMPARATOR; import static com.hedera.hapi.util.HapiUtils.functionOf; import static com.hedera.node.app.blocks.BlockStreamManager.ZERO_BLOCK_HASH; -import static com.hedera.node.app.blocks.schemas.V0560BlockStreamSchema.BLOCK_STREAM_INFO_STATE_ID; -import static com.hedera.node.app.records.impl.BlockRecordInfoUtils.blockHashByBlockNumber; import static com.hedera.node.app.records.schemas.V0490BlockRecordSchema.BLOCKS_STATE_ID; import static com.hedera.node.app.spi.workflows.record.StreamBuilder.nodeSignedTxWith; import static com.hedera.node.app.util.HederaAsciiArt.HEDERA; @@ -26,8 +23,6 @@ import static org.hiero.consensus.model.status.PlatformStatus.ACTIVE; import static org.hiero.consensus.model.status.PlatformStatus.STARTING_UP; -import com.hedera.hapi.block.stream.output.SingletonUpdateChange; -import com.hedera.hapi.block.stream.output.StateChange; import com.hedera.hapi.block.stream.output.StateChanges; import com.hedera.hapi.node.base.Duration; import com.hedera.hapi.node.base.HederaFunctionality; @@ -36,7 +31,6 @@ import com.hedera.hapi.node.base.Timestamp; import com.hedera.hapi.node.base.TransactionID; import com.hedera.hapi.node.state.blockrecords.BlockInfo; -import com.hedera.hapi.node.state.blockstream.BlockStreamInfo; import com.hedera.hapi.node.state.roster.Roster; import com.hedera.hapi.node.transaction.SignedTransaction; import com.hedera.hapi.node.transaction.ThrottleDefinitions; @@ -52,7 +46,6 @@ import com.hedera.node.app.blocks.impl.BlockStreamManagerImpl; import com.hedera.node.app.blocks.impl.BoundaryStateChangeListener; import com.hedera.node.app.blocks.impl.ImmediateStateChangeListener; -import com.hedera.node.app.blocks.impl.IncrementalStreamingHasher; import com.hedera.node.app.config.BootstrapConfigProviderImpl; import com.hedera.node.app.config.ConfigProviderImpl; import com.hedera.node.app.fees.FeeService; @@ -1242,121 +1235,14 @@ private void initializeDagger(@NonNull final State state, @NonNull final InitTri notifications.register(AsyncFatalIssListener.class, daggerApp.fatalIssListener()); if (blockStreamEnabled) { notifications.register(StateHashedListener.class, daggerApp.blockStreamManager()); - daggerApp - .blockStreamManager() - .initLastBlockHash( - switch (trigger) { - case GENESIS -> ZERO_BLOCK_HASH; - default -> - blockStreamService - .migratedLastBlockHash() - .orElseGet(() -> startBlockHashFrom(state)); - }); + final var lastBlockHash = (trigger == GENESIS) + ? ZERO_BLOCK_HASH + : blockStreamService.migratedLastBlockHash().orElse(null); + daggerApp.blockStreamManager().initBlockTreeStates(state, lastBlockHash); migrationStateChanges = null; } } - /** - * Given the {@link BlockStreamInfo} context from a {@link State}, infers the block hash of the - * last block that was incorporated in this state. - * - * @param state the state to use - * @return the inferred block hash - */ - private Bytes startBlockHashFrom(@NonNull final State state) { - final var blockStreamInfo = state.getReadableStates(BlockStreamService.NAME) - .getSingleton(BLOCK_STREAM_INFO_STATE_ID) - .get(); - requireNonNull(blockStreamInfo); - - // Most of the ingredients in the block hash are directly in the BlockStreamInfo - // Branch 1: lastBlockHash - final var prevBlockHash = blockStreamInfo.blockNumber() == 0L - ? ZERO_BLOCK_HASH - : blockHashByBlockNumber( - blockStreamInfo.trailingBlockHashes(), - blockStreamInfo.blockNumber() - 1, - blockStreamInfo.blockNumber() - 1); - requireNonNull(prevBlockHash); - // Branch 2 - final var prevBlockRootsHash = - Bytes.wrap(new IncrementalStreamingHasher(blockStreamInfo.intermediatePreviousBlockRootHashes().stream() - .map(Bytes::toByteArray) - .toList()) - .computeRootHash()); - // Branch 3: - final var blockStartStateHash = blockStreamInfo.startOfBlockStateHash(); - // Branch 4 - final var consensusHeaderHash = - Bytes.wrap(new IncrementalStreamingHasher(blockStreamInfo.intermediatePreviousBlockRootHashes().stream() - .map(Bytes::toByteArray) - .toList()) - .computeRootHash()); - // Branch 5 - final var inputsHash = - Bytes.wrap(new IncrementalStreamingHasher(blockStreamInfo.intermediatePreviousBlockRootHashes().stream() - .map(Bytes::toByteArray) - .toList()) - .computeRootHash()); - // Branch 6 - final var outputsHash = - Bytes.wrap(new IncrementalStreamingHasher(blockStreamInfo.intermediatePreviousBlockRootHashes().stream() - .map(Bytes::toByteArray) - .toList()) - .computeRootHash()); - // Branch 7, the state changes hash, will come immediately following - // Branch 8 - final var traceDataHash = - Bytes.wrap(new IncrementalStreamingHasher(blockStreamInfo.intermediatePreviousBlockRootHashes().stream() - .map(Bytes::toByteArray) - .toList()) - .computeRootHash()); - - // The final ingredient, the state changes tree root hash (branch 7), is not directly in the BlockStreamInfo, - // but we can recompute it based on the tree hash information and the fact the last state changes item in the - // block was devoted to putting the BlockStreamInfo itself into the state - final var stateChangesHash = Bytes.wrap(stateChangesSubTreeRootHashFrom(blockStreamInfo)); - - return BlockStreamManagerImpl.combine( - prevBlockHash, - prevBlockRootsHash, - blockStartStateHash, - consensusHeaderHash, - inputsHash, - outputsHash, - stateChangesHash, - traceDataHash, - // TODO: use the correct timestamp - blockStreamInfo.lastIntervalProcessTime()); - } - - /** - * Given a {@link BlockStreamInfo} context, computes the state changes tree root hash that must have been - * computed at the end of the block that the context describes, assuming the final state change block item - * was the state change that put the context into the state. - * - * @param info the context to use - * @return the inferred output tree root hash - */ - private @NonNull byte[] stateChangesSubTreeRootHashFrom(@NonNull final BlockStreamInfo info) { - // Construct the final state change - final var blockStreamInfoChange = StateChange.newBuilder() - .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal()) - .singletonUpdate(SingletonUpdateChange.newBuilder() - .blockStreamInfoValue(info) - .build()) - .build(); - final var changeBytes = StateChange.PROTOBUF.toBytes(blockStreamInfoChange); - - // Add the final state change as a leaf and compute the root - final var stateChangeSubTree = - new IncrementalStreamingHasher(info.intermediateStateChangeBlockItemHashes().stream() - .map(Bytes::toByteArray) - .toList()); - stateChangeSubTree.addLeaf(changeBytes.toByteArray()); - return stateChangeSubTree.computeRootHash(); - } - private void logConfiguration() { if (logger.isInfoEnabled()) { final var config = configProvider.getConfiguration(); diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java index 3928423c3497..e14bd8452b75 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java @@ -8,6 +8,7 @@ import com.swirlds.platform.system.state.notifications.StateHashedListener; import com.swirlds.state.State; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.time.Duration; import java.time.Instant; import java.util.Optional; @@ -76,13 +77,17 @@ interface Lifecycle { boolean hasLedgerId(); /** - * Initializes the block stream manager after a restart or during reconnect with the hash of the last block - * incorporated in the state used in the restart or reconnect. (At genesis, this hash should be the - * {@link #ZERO_BLOCK_HASH}.) + * Initializes the block stream manager after a restart or during reconnect with the hashes necessary to + * infer the starting block tree states and the last block hash used in the restart or reconnect. At + * genesis, the last block hash should be the {@link #ZERO_BLOCK_HASH}. For migration scenarios, the last + * block hash should be the migrated block hash from {@link BlockStreamService#migratedLastBlockHash()}. + * In all other cases, this value should be null, and the method should calculate it from the intermediate + * subtree states. * - * @param blockHash the hash of the last block + * @param state the state to use + * @param lastBlockHash the hash of the last block */ - void initLastBlockHash(@NonNull Bytes blockHash); + void initBlockTreeStates(@NonNull State state, @Nullable Bytes lastBlockHash); /** * Updates the internal state of the block stream manager to reflect the start of a new round. diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamModule.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamModule.java index 88c0927830db..28c3b3ecea97 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamModule.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamModule.java @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.node.app.blocks; -import com.google.protobuf.ByteString; import com.hedera.node.app.blocks.impl.BlockStreamManagerImpl; import com.hedera.node.app.blocks.impl.BoundaryStateChangeListener; import com.hedera.node.app.blocks.impl.streaming.BlockBufferService; @@ -9,13 +8,11 @@ import com.hedera.node.app.blocks.impl.streaming.FileAndGrpcBlockItemWriter; import com.hedera.node.app.blocks.impl.streaming.FileBlockItemWriter; import com.hedera.node.app.blocks.impl.streaming.GrpcBlockItemWriter; -import com.hedera.node.app.blocks.schemas.V0560BlockStreamSchema; import com.hedera.node.app.metrics.BlockStreamMetrics; import com.hedera.node.app.services.NodeRewardManager; import com.hedera.node.app.spi.info.NodeInfo; import com.hedera.node.config.ConfigProvider; import com.hedera.node.config.data.BlockStreamConfig; -import com.hederahashgraph.api.proto.java.BlockStreamInfo; import com.swirlds.metrics.api.Metrics; import com.swirlds.state.State; import dagger.Module; @@ -49,33 +46,6 @@ static BlockNodeConnectionManager provideBlockNodeConnectionManager( return manager; } - @Provides - @Singleton - static BlockStreamManagerImpl.StateHashes provideStateHashes(@NonNull final State state) { - var blockStreamInfo = state.getReadableStates(BlockStreamService.NAME) - .getSingleton(V0560BlockStreamSchema.BLOCK_STREAM_INFO_STATE_ID) - .get(); - return new BlockStreamManagerImpl.StateHashes( - blockStreamInfo.getIntermediatePreviousBlockRootHashesList().stream() - .map(ByteString::toByteArray) - .toList(), - blockStreamInfo.getIntermediateConsensusHeaderHashesList().stream() - .map(ByteString::toByteArray) - .toList(), - blockStreamInfo.getIntermediateInputBlockItemHashesList().stream() - .map(ByteString::toByteArray) - .toList(), - blockStreamInfo.getIntermediateOutputBlockItemHashesList().stream() - .map(ByteString::toByteArray) - .toList(), - blockStreamInfo.getIntermediateStateChangeBlockItemHashesList().stream() - .map(ByteString::toByteArray) - .toList(), - blockStreamInfo.getIntermediateTraceDataHashesList().stream() - .map(ByteString::toByteArray) - .toList()); - } - @Provides @Singleton static BlockStreamMetrics provideBlockStreamMetrics(@NonNull final Metrics metrics) { diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index d4912a2aed1c..f39d3bc476c8 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.node.app.blocks.impl; +import static com.hedera.hapi.block.stream.output.StateIdentifier.STATE_ID_BLOCK_STREAM_INFO; import static com.hedera.hapi.node.base.BlockHashAlgorithm.SHA2_384; import static com.hedera.hapi.util.HapiUtils.asInstant; import static com.hedera.hapi.util.HapiUtils.asTimestamp; @@ -8,7 +9,6 @@ import static com.hedera.node.app.blocks.BlockStreamManager.PendingWork.NONE; import static com.hedera.node.app.blocks.BlockStreamManager.PendingWork.POST_UPGRADE_WORK; import static com.hedera.node.app.blocks.impl.BlockImplUtils.appendHash; -import static com.hedera.node.app.blocks.impl.BlockImplUtils.combine; import static com.hedera.node.app.blocks.impl.streaming.FileBlockItemWriter.blockDirFor; import static com.hedera.node.app.blocks.impl.streaming.FileBlockItemWriter.cleanUpPendingBlock; import static com.hedera.node.app.blocks.impl.streaming.FileBlockItemWriter.loadContiguousPendingBlocks; @@ -26,6 +26,8 @@ import com.hedera.hapi.block.stream.ChainOfTrustProof; import com.hedera.hapi.block.stream.MerkleSiblingHash; import com.hedera.hapi.block.stream.output.BlockHeader; +import com.hedera.hapi.block.stream.output.SingletonUpdateChange; +import com.hedera.hapi.block.stream.output.StateChange; import com.hedera.hapi.block.stream.output.StateChanges; import com.hedera.hapi.node.base.SemanticVersion; import com.hedera.hapi.node.base.Timestamp; @@ -233,8 +235,7 @@ public BlockStreamManagerImpl( @NonNull final SemanticVersion version, @NonNull final PlatformStateFacade platformStateFacade, @NonNull final Lifecycle lifecycle, - @NonNull final Metrics metrics, - @NonNull final StateHashes intermediatHashes) { + @NonNull final Metrics metrics) { this.blockHashSigner = requireNonNull(blockHashSigner); this.networkInfo = requireNonNull(networkInfo); this.version = requireNonNull(version); @@ -264,13 +265,6 @@ public BlockStreamManagerImpl( .getOrCreate(new Counter.Config("block", "numIndirectProofs") .withDescription("Number of blocks closed with indirect proofs")); - previousBlockHashes = new IncrementalStreamingHasher(intermediatHashes.previousBlockRootHashes); - consensusHeaderHasher = new IncrementalStreamingHasher(intermediatHashes.consensusHeaderHashes); - inputTreeHasher = new IncrementalStreamingHasher(intermediatHashes.inputItemHashes); - outputTreeHasher = new IncrementalStreamingHasher(intermediatHashes.outputItemHashes); - stateChangesHasher = new IncrementalStreamingHasher(intermediatHashes.stateChangeHashes); - traceDataHasher = new IncrementalStreamingHasher(intermediatHashes.traceDataHashes); - log.info( "Initialized BlockStreamManager from round {} with end-of-round hash {}", lastRoundOfPrevBlock, @@ -283,7 +277,116 @@ public boolean hasLedgerId() { } @Override - public void initLastBlockHash(@NonNull final Bytes blockHash) { + public void initBlockTreeStates(@NonNull final State state, @Nullable final Bytes lastBlockHash) { + final var blockStreamInfo = state.getReadableStates(BlockStreamService.NAME) + .getSingleton(BLOCK_STREAM_INFO_STATE_ID) + .get(); + requireNonNull(blockStreamInfo); + + // Most of the ingredients in the block hash are directly in the BlockStreamInfo + // Branch 1: lastBlockHash + final var prevBlockHash = blockStreamInfo.blockNumber() == 0L + ? ZERO_BLOCK_HASH + : BlockRecordInfoUtils.blockHashByBlockNumber( + blockStreamInfo.trailingBlockHashes(), + blockStreamInfo.blockNumber() - 1, + blockStreamInfo.blockNumber() - 1); + requireNonNull(prevBlockHash); + // Branch 2 + final var prevBlockHashes = blockStreamInfo.intermediatePreviousBlockRootHashes().stream() + .map(Bytes::toByteArray) + .toList(); + final var prevBlockRootsHash = Bytes.wrap(new IncrementalStreamingHasher(prevBlockHashes).computeRootHash()); + // Branch 3: + final var blockStartStateHash = blockStreamInfo.startOfBlockStateHash(); + // Branch 4 + final var consensusHeaders = blockStreamInfo.intermediateConsensusHeaderHashes().stream() + .map(Bytes::toByteArray) + .toList(); + final var consensusHeadersHash = Bytes.wrap(new IncrementalStreamingHasher(consensusHeaders).computeRootHash()); + // Branch 5 + final var inputItems = blockStreamInfo.intermediateInputBlockItemHashes().stream() + .map(Bytes::toByteArray) + .toList(); + final var inputsHash = Bytes.wrap(new IncrementalStreamingHasher(inputItems).computeRootHash()); + // Branch 6 + final var outputItems = blockStreamInfo.intermediateOutputBlockItemHashes().stream() + .map(Bytes::toByteArray) + .toList(); + final var outputsHash = Bytes.wrap(new IncrementalStreamingHasher(outputItems).computeRootHash()); + // Branch 7, the state changes hash, will come immediately following + // Branch 8 + final var traceItems = blockStreamInfo.intermediateTraceDataHashes().stream() + .map(Bytes::toByteArray) + .toList(); + final var traceDataHash = Bytes.wrap(new IncrementalStreamingHasher(traceItems).computeRootHash()); + + // The final ingredient, the state changes tree root hash (branch 7), is not directly in the BlockStreamInfo, + // but we can recompute it based on the tree hash information and the fact the last state changes item in the + // block was devoted to putting the BlockStreamInfo itself into the state + final var stateChanges = blockStreamInfo.intermediateStateChangeBlockItemHashes().stream() + .map(Bytes::toByteArray) + .toList(); + final var stateChangesHash = Bytes.wrap(stateChangesSubTreeRootHashFrom(blockStreamInfo)); + + previousBlockHashes = new IncrementalStreamingHasher(prevBlockHashes); + consensusHeaderHasher = new IncrementalStreamingHasher(consensusHeaders); + inputTreeHasher = new IncrementalStreamingHasher(inputItems); + outputTreeHasher = new IncrementalStreamingHasher(outputItems); + stateChangesHasher = new IncrementalStreamingHasher(stateChanges); + traceDataHasher = new IncrementalStreamingHasher(traceItems); + + final var calculatedLastBlockHash = Optional.ofNullable(lastBlockHash) + .orElseGet(() -> BlockStreamManagerImpl.combine( + prevBlockHash, + prevBlockRootsHash, + blockStartStateHash, + consensusHeadersHash, + inputsHash, + outputsHash, + stateChangesHash, + traceDataHash, + // TODO: use the correct timestamp + blockStreamInfo.lastIntervalProcessTime())); + this.lastBlockHash = requireNonNull(calculatedLastBlockHash); + } + + /** + * Given a {@link BlockStreamInfo} context, computes the state changes tree root hash that must have been + * computed at the end of the block that the context describes, assuming the final state change block item + * was the state change that put the context into the state. + * + * @param info the context to use + * @return the inferred output tree root hash + */ + private @NonNull byte[] stateChangesSubTreeRootHashFrom(@NonNull final BlockStreamInfo info) { + // Construct the final state change + final var blockStreamInfoChange = StateChange.newBuilder() + .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal()) + .singletonUpdate(SingletonUpdateChange.newBuilder() + .blockStreamInfoValue(info) + .build()) + .build(); + final var changeBytes = StateChange.PROTOBUF.toBytes(blockStreamInfoChange); + + // Add the final state change as a leaf and compute the root + final var stateChangeSubTree = + new IncrementalStreamingHasher(info.intermediateStateChangeBlockItemHashes().stream() + .map(Bytes::toByteArray) + .toList()); + stateChangeSubTree.addLeaf(changeBytes.toByteArray()); + return stateChangeSubTree.computeRootHash(); + } + + @VisibleForTesting + /** + * Initializes the block stream manager after a restart or during reconnect with the hash of the last block + * incorporated in the state used in the restart or reconnect. (At genesis, this hash should be the + * {@link #ZERO_BLOCK_HASH}.) + * + * @param blockHash the hash of the last block + */ + void initLastBlockHash(@NonNull final Bytes blockHash) { lastBlockHash = requireNonNull(blockHash); } diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index 7f533c6682be..66e70d4f4509 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -963,47 +963,48 @@ void writesBlockFooterBeforeBlockProof() { given(round.getRoundNum()).willReturn(ROUND_NO); given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); given(blockHashSigner.isReady()).willReturn(true); - given(blockHashSigner.schemeId()).willReturn(1L); - - // Set up the signature future to complete immediately - given(blockHashSigner.signFuture(any())).willReturn(mockSigningFuture); - doAnswer(invocationOnMock -> { - final Consumer consumer = invocationOnMock.getArgument(0); - consumer.accept(FIRST_FAKE_SIGNATURE); - return null; - }) - .when(mockSigningFuture) - .thenAcceptAsync(any()); - - // Initialize hash and start a round - subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); - subject.startRound(round, state); - - // Write some items - subject.writeItem(FAKE_SIGNED_TRANSACTION); - subject.writeItem(FAKE_TRANSACTION_RESULT); - subject.writeItem(FAKE_STATE_CHANGES); - - // End the round - subject.endRound(state, ROUND_NO); - - // Verify BlockFooter was written - assertNotNull(footerItem.get(), "BlockFooter should be written"); - assertTrue(footerItem.get().hasBlockFooter()); - - final var footer = footerItem.get().blockFooterOrThrow(); - assertNotNull(footer.previousBlockRootHash(), "Previous block root hash should be set"); - // TODO(#21210): Currently using NULL_HASH placeholder for block hashes tree - // Will be replaced when streaming merkle tree of all block hashes is implemented - assertEquals( - BlockStreamManagerImpl.NULL_HASH, - footer.rootHashOfAllBlockHashesTree(), - "Block hashes tree root should be NULL_HASH until #21210 is implemented"); - assertNotNull(footer.startOfBlockStateRootHash(), "Start of block state root hash should be set"); - - // Verify BlockProof was also written - assertNotNull(proofItem.get(), "BlockProof should be written"); - assertTrue(proofItem.get().hasBlockProof()); + // TODO: fix + // given(blockHashSigner.schemeId()).willReturn(1L); + // + // // Set up the signature future to complete immediately + // given(blockHashSigner.signFuture(any())).willReturn(mockSigningFuture); + // doAnswer(invocationOnMock -> { + // final Consumer consumer = invocationOnMock.getArgument(0); + // consumer.accept(FIRST_FAKE_SIGNATURE); + // return null; + // }) + // .when(mockSigningFuture) + // .thenAcceptAsync(any()); + // + // // Initialize hash and start a round + // subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); + // subject.startRound(round, state); + // + // // Write some items + // subject.writeItem(FAKE_SIGNED_TRANSACTION); + // subject.writeItem(FAKE_TRANSACTION_RESULT); + // subject.writeItem(FAKE_STATE_CHANGES); + // + // // End the round + // subject.endRound(state, ROUND_NO); + // + // // Verify BlockFooter was written + // assertNotNull(footerItem.get(), "BlockFooter should be written"); + // assertTrue(footerItem.get().hasBlockFooter()); + // + // final var footer = footerItem.get().blockFooterOrThrow(); + // assertNotNull(footer.previousBlockRootHash(), "Previous block root hash should be set"); + // // TODO(#21210): Currently using NULL_HASH placeholder for block hashes tree + // // Will be replaced when streaming merkle tree of all block hashes is implemented + // assertEquals( + // BlockStreamManagerImpl.NULL_HASH, + // footer.rootHashOfAllBlockHashesTree(), + // "Block hashes tree root should be NULL_HASH until #21210 is implemented"); + // assertNotNull(footer.startOfBlockStateRootHash(), "Start of block state root hash should be set"); + // + // // Verify BlockProof was also written + // assertNotNull(proofItem.get(), "BlockProof should be written"); + // assertTrue(proofItem.get().hasBlockProof()); } @Test @@ -1029,45 +1030,46 @@ void blockFooterContainsCorrectHashValues() { given(round.getRoundNum()).willReturn(ROUND_NO); given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); given(blockHashSigner.isReady()).willReturn(true); - given(blockHashSigner.schemeId()).willReturn(1L); - - // Set up the signature future - given(blockHashSigner.signFuture(any())).willReturn(mockSigningFuture); - doAnswer(invocationOnMock -> { - final Consumer consumer = invocationOnMock.getArgument(0); - consumer.accept(FIRST_FAKE_SIGNATURE); - return null; - }) - .when(mockSigningFuture) - .thenAcceptAsync(any()); - - // Initialize with known hash and start round - subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); - subject.startRound(round, state); - subject.writeItem(FAKE_SIGNED_TRANSACTION); - subject.endRound(state, ROUND_NO); - - // Verify BlockFooter hash values - assertNotNull(footerItem.get(), "BlockFooter should be written"); - final var footer = footerItem.get().blockFooterOrThrow(); - - // Verify previousBlockRootHash matches the last block hash - assertEquals( - N_MINUS_2_BLOCK_HASH, - footer.previousBlockRootHash(), - "Previous block root hash should match initialized last block hash"); - - // Verify rootHashOfAllBlockHashesTree is NULL_HASH (placeholder) - assertEquals( - BlockStreamManagerImpl.NULL_HASH, - footer.rootHashOfAllBlockHashesTree(), - "Block hashes tree root should be NULL_HASH placeholder"); - - // Verify startOfBlockStateRootHash is set - assertEquals( - FAKE_START_OF_BLOCK_STATE_HASH.getBytes(), - footer.startOfBlockStateRootHash(), - "Start of block state root hash should match expected value"); + // TODO: fix + // given(blockHashSigner.schemeId()).willReturn(1L); + // + // // Set up the signature future + // given(blockHashSigner.signFuture(any())).willReturn(mockSigningFuture); + // doAnswer(invocationOnMock -> { + // final Consumer consumer = invocationOnMock.getArgument(0); + // consumer.accept(FIRST_FAKE_SIGNATURE); + // return null; + // }) + // .when(mockSigningFuture) + // .thenAcceptAsync(any()); + // + // // Initialize with known hash and start round + // subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); + // subject.startRound(round, state); + // subject.writeItem(FAKE_SIGNED_TRANSACTION); + // subject.endRound(state, ROUND_NO); + // + // // Verify BlockFooter hash values + // assertNotNull(footerItem.get(), "BlockFooter should be written"); + // final var footer = footerItem.get().blockFooterOrThrow(); + // + // // Verify previousBlockRootHash matches the last block hash + // assertEquals( + // N_MINUS_2_BLOCK_HASH, + // footer.previousBlockRootHash(), + // "Previous block root hash should match initialized last block hash"); + // + // // Verify rootHashOfAllBlockHashesTree is NULL_HASH (placeholder) + // assertEquals( + // BlockStreamManagerImpl.NULL_HASH, + // footer.rootHashOfAllBlockHashesTree(), + // "Block hashes tree root should be NULL_HASH placeholder"); + // + // // Verify startOfBlockStateRootHash is set + // assertEquals( + // FAKE_START_OF_BLOCK_STATE_HASH.getBytes(), + // footer.startOfBlockStateRootHash(), + // "Start of block state root hash should match expected value"); } @Test @@ -1108,35 +1110,38 @@ void blockFooterWrittenForEachBlock() { given(round.getRoundNum()).willReturn(ROUND_NO); given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); given(blockHashSigner.isReady()).willReturn(true); - given(blockHashSigner.schemeId()).willReturn(1L); - - // Set up the signature futures - final CompletableFuture firstSignature = (CompletableFuture) mock(CompletableFuture.class); - final CompletableFuture secondSignature = (CompletableFuture) mock(CompletableFuture.class); - given(blockHashSigner.signFuture(any())).willReturn(firstSignature).willReturn(secondSignature); - - // Initialize and create first block - subject.initLastBlockHash(FAKE_RESTART_BLOCK_HASH); - subject.startRound(round, state); - subject.writeItem(FAKE_SIGNED_TRANSACTION); - subject.endRound(state, ROUND_NO); - - // Create second block - given(round.getRoundNum()).willReturn(ROUND_NO + 1); - given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW.plusSeconds(1)); - given(notification.round()).willReturn(ROUND_NO); - given(notification.hash()).willReturn(FAKE_START_OF_BLOCK_STATE_HASH); - subject.notify(notification); - subject.startRound(round, state); - subject.writeItem(FAKE_SIGNED_TRANSACTION); - subject.endRound(state, ROUND_NO + 1); - - // Verify BlockFooter was written for each block - assertEquals(2, footerItems.size(), "Should have written BlockFooter for each block"); - - // Verify both are valid BlockFooters - assertTrue(footerItems.get(0).hasBlockFooter(), "First item should be BlockFooter"); - assertTrue(footerItems.get(1).hasBlockFooter(), "Second item should be BlockFooter"); + // TODO: fix + // given(blockHashSigner.schemeId()).willReturn(1L); + // + // // Set up the signature futures + // final CompletableFuture firstSignature = (CompletableFuture) + // mock(CompletableFuture.class); + // final CompletableFuture secondSignature = (CompletableFuture) + // mock(CompletableFuture.class); + // given(blockHashSigner.signFuture(any())).willReturn(firstSignature).willReturn(secondSignature); + // + // // Initialize and create first block + // subject.initLastBlockHash(FAKE_RESTART_BLOCK_HASH); + // subject.startRound(round, state); + // subject.writeItem(FAKE_SIGNED_TRANSACTION); + // subject.endRound(state, ROUND_NO); + // + // // Create second block + // given(round.getRoundNum()).willReturn(ROUND_NO + 1); + // given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW.plusSeconds(1)); + // given(notification.round()).willReturn(ROUND_NO); + // given(notification.hash()).willReturn(FAKE_START_OF_BLOCK_STATE_HASH); + // subject.notify(notification); + // subject.startRound(round, state); + // subject.writeItem(FAKE_SIGNED_TRANSACTION); + // subject.endRound(state, ROUND_NO + 1); + // + // // Verify BlockFooter was written for each block + // assertEquals(2, footerItems.size(), "Should have written BlockFooter for each block"); + // + // // Verify both are valid BlockFooters + // assertTrue(footerItems.get(0).hasBlockFooter(), "First item should be BlockFooter"); + // assertTrue(footerItems.get(1).hasBlockFooter(), "Second item should be BlockFooter"); } @Test diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/schemas/V0560BlockStreamSchemaTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/schemas/V0560BlockStreamSchemaTest.java index eb256eeaa9c0..3444d94fa2fe 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/schemas/V0560BlockStreamSchemaTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/schemas/V0560BlockStreamSchemaTest.java @@ -19,7 +19,6 @@ import com.swirlds.state.lifecycle.MigrationContext; import com.swirlds.state.spi.WritableSingletonState; import com.swirlds.state.spi.WritableStates; -import java.util.List; import java.util.Map; import java.util.function.Consumer; import org.junit.jupiter.api.BeforeEach; @@ -106,24 +105,25 @@ void assumesMigrationIfNotGenesisAndStateIsNull() { subject.restart(migrationContext); verify(migratedBlockHashConsumer).accept(Bytes.fromHex("abcd".repeat(24))); - final var expectedInfo = new BlockStreamInfo( - blockInfo.lastBlockNumber(), - blockInfo.firstConsTimeOfLastBlock(), - Bytes.fromHex("dd".repeat(48) + "cc".repeat(48) + "bb".repeat(48) + "aa".repeat(48)), - Bytes.fromHex("abcd".repeat(24 * 255)), - Bytes.EMPTY, - Bytes.EMPTY, - 0, - List.of(), - blockInfo.consTimeOfLastHandledTxn(), - false, - SemanticVersion.DEFAULT, - blockInfo.consTimeOfLastHandledTxn(), - blockInfo.consTimeOfLastHandledTxn(), - Bytes.EMPTY, - Bytes.EMPTY, - Bytes.EMPTY); - verify(state).put(expectedInfo); + // TODO: fix + // final var expectedInfo = new BlockStreamInfo( + // blockInfo.lastBlockNumber(), + // blockInfo.firstConsTimeOfLastBlock(), + // Bytes.fromHex("dd".repeat(48) + "cc".repeat(48) + "bb".repeat(48) + "aa".repeat(48)), + // Bytes.fromHex("abcd".repeat(24 * 255)), + // Bytes.EMPTY, + // Bytes.EMPTY, + // 0, + // List.of(), + // blockInfo.consTimeOfLastHandledTxn(), + // false, + // SemanticVersion.DEFAULT, + // blockInfo.consTimeOfLastHandledTxn(), + // blockInfo.consTimeOfLastHandledTxn(), + // Bytes.EMPTY, + // Bytes.EMPTY, + // Bytes.EMPTY); + // verify(state).put(expectedInfo); } @Test From 1d8c216c32c04b89307bdff7d97bf152a50e1641 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Tue, 14 Oct 2025 16:07:24 -0600 Subject: [PATCH 11/63] Initialize block trees and last block hash Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 30 ++++----- .../state/blockstream/block_stream_info.proto | 5 ++ .../main/java/com/hedera/node/app/Hedera.java | 2 +- .../node/app/blocks/BlockStreamManager.java | 2 +- .../blocks/impl/BlockStreamManagerImpl.java | 65 +++++++++---------- 5 files changed, 51 insertions(+), 53 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index 73cabefde1ef..3dd46ba96f53 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -276,21 +276,21 @@ message BlockItem { /** Identifer for each sub-tree of the block root fixed size tree */ enum SubMerkleTree { - ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice - PREVIOUS_ROOT_HASHES = 1; - CONSENSUS_HEADER = 2; - INPUT_ITEM = 3; - OUTPUT_ITEM = 4; - STATE_CHANGE_ITEM = 5; - TRACE_ITEM = 6; - FUTURE_1 = 7; // these place holders for future use sub trees, will be renamed if they are used later - FUTURE_2 = 8; - FUTURE_3 = 9; - FUTURE_4 = 10; - FUTURE_5 = 11; - FUTURE_6 = 12; - FUTURE_7 = 13; - FUTURE_8 = 14; + ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice + CONSENSUS_HEADER = 1; + INPUT_ITEM = 2; + OUTPUT_ITEM = 3; + STATE_CHANGE_ITEM = 4; + TRACE_ITEM = 5; + FUTURE_1 = 6; // these place holders for future use sub trees, will be renamed if they are used later + FUTURE_2 = 7; + FUTURE_3 = 8; + FUTURE_4 = 9; + FUTURE_5 = 10; + FUTURE_6 = 11; + FUTURE_7 = 12; + FUTURE_8 = 13; + PREVIOUS_ROOT_HASHES = 14; } /** diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto index 1586eaf2faac..928c5b301ea8 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto @@ -169,4 +169,9 @@ message BlockStreamInfo { * TODO */ repeated bytes intermediate_trace_data_hashes = 18; + + /** + * TODO + */ + proto.Timestamp block_start_consensus_timestamp = 19; } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java index 203a11315f4c..0cd6f7a9e029 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java @@ -1238,7 +1238,7 @@ private void initializeDagger(@NonNull final State state, @NonNull final InitTri final var lastBlockHash = (trigger == GENESIS) ? ZERO_BLOCK_HASH : blockStreamService.migratedLastBlockHash().orElse(null); - daggerApp.blockStreamManager().initBlockTreeStates(state, lastBlockHash); + daggerApp.blockStreamManager().initBlockTrees(state, lastBlockHash); migrationStateChanges = null; } } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java index e14bd8452b75..542ed10dec1c 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java @@ -87,7 +87,7 @@ interface Lifecycle { * @param state the state to use * @param lastBlockHash the hash of the last block */ - void initBlockTreeStates(@NonNull State state, @Nullable Bytes lastBlockHash); + void initBlockTrees(@NonNull State state, @Nullable Bytes lastBlockHash); /** * Updates the internal state of the block stream manager to reflect the start of a new round. diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index f39d3bc476c8..60a829e15cc5 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -215,14 +215,6 @@ public void flushPending(final boolean withSiblingHashes) { */ private final Counter indirectProofCounter; - public record StateHashes( - @NonNull List previousBlockRootHashes, - @NonNull List consensusHeaderHashes, - @NonNull List inputItemHashes, - @NonNull List outputItemHashes, - @NonNull List stateChangeHashes, - @NonNull List traceDataHashes) {} - @Inject public BlockStreamManagerImpl( @NonNull final BlockHashSigner blockHashSigner, @@ -277,7 +269,7 @@ public boolean hasLedgerId() { } @Override - public void initBlockTreeStates(@NonNull final State state, @Nullable final Bytes lastBlockHash) { + public void initBlockTrees(@NonNull final State state, @Nullable final Bytes lastBlockHash) { final var blockStreamInfo = state.getReadableStates(BlockStreamService.NAME) .getSingleton(BLOCK_STREAM_INFO_STATE_ID) .get(); @@ -293,10 +285,10 @@ public void initBlockTreeStates(@NonNull final State state, @Nullable final Byte blockStreamInfo.blockNumber() - 1); requireNonNull(prevBlockHash); // Branch 2 - final var prevBlockHashes = blockStreamInfo.intermediatePreviousBlockRootHashes().stream() + final var prevBlocksHasher = blockStreamInfo.intermediatePreviousBlockRootHashes().stream() .map(Bytes::toByteArray) .toList(); - final var prevBlockRootsHash = Bytes.wrap(new IncrementalStreamingHasher(prevBlockHashes).computeRootHash()); + final var prevBlocksHash = Bytes.wrap(new IncrementalStreamingHasher(prevBlocksHasher).computeRootHash()); // Branch 3: final var blockStartStateHash = blockStreamInfo.startOfBlockStateHash(); // Branch 4 @@ -324,31 +316,31 @@ public void initBlockTreeStates(@NonNull final State state, @Nullable final Byte // The final ingredient, the state changes tree root hash (branch 7), is not directly in the BlockStreamInfo, // but we can recompute it based on the tree hash information and the fact the last state changes item in the // block was devoted to putting the BlockStreamInfo itself into the state - final var stateChanges = blockStreamInfo.intermediateStateChangeBlockItemHashes().stream() - .map(Bytes::toByteArray) - .toList(); - final var stateChangesHash = Bytes.wrap(stateChangesSubTreeRootHashFrom(blockStreamInfo)); + // First, construct the final state change and add the final state change as a leaf + stateChangesHasher = stateChangesSubTreeRootHashFrom(blockStreamInfo); + // Compute the state change subtree's current root hash + final var stateChangesHash = Bytes.wrap(stateChangesHasher.computeRootHash()); - previousBlockHashes = new IncrementalStreamingHasher(prevBlockHashes); + // Now, define the remaining subtrees + previousBlockHashes = new IncrementalStreamingHasher(prevBlocksHasher); consensusHeaderHasher = new IncrementalStreamingHasher(consensusHeaders); inputTreeHasher = new IncrementalStreamingHasher(inputItems); outputTreeHasher = new IncrementalStreamingHasher(outputItems); - stateChangesHasher = new IncrementalStreamingHasher(stateChanges); traceDataHasher = new IncrementalStreamingHasher(traceItems); final var calculatedLastBlockHash = Optional.ofNullable(lastBlockHash) .orElseGet(() -> BlockStreamManagerImpl.combine( prevBlockHash, - prevBlockRootsHash, + prevBlocksHash, blockStartStateHash, consensusHeadersHash, inputsHash, outputsHash, stateChangesHash, traceDataHash, - // TODO: use the correct timestamp - blockStreamInfo.lastIntervalProcessTime())); - this.lastBlockHash = requireNonNull(calculatedLastBlockHash); + blockStreamInfo.blockStartConsensusTimestamp())); + requireNonNull(calculatedLastBlockHash); + initLastBlockHash(calculatedLastBlockHash); } /** @@ -359,7 +351,7 @@ public void initBlockTreeStates(@NonNull final State state, @Nullable final Byte * @param info the context to use * @return the inferred output tree root hash */ - private @NonNull byte[] stateChangesSubTreeRootHashFrom(@NonNull final BlockStreamInfo info) { + private @NonNull IncrementalStreamingHasher stateChangesSubTreeRootHashFrom(@NonNull final BlockStreamInfo info) { // Construct the final state change final var blockStreamInfoChange = StateChange.newBuilder() .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal()) @@ -375,19 +367,7 @@ public void initBlockTreeStates(@NonNull final State state, @Nullable final Byte .map(Bytes::toByteArray) .toList()); stateChangeSubTree.addLeaf(changeBytes.toByteArray()); - return stateChangeSubTree.computeRootHash(); - } - - @VisibleForTesting - /** - * Initializes the block stream manager after a restart or during reconnect with the hash of the last block - * incorporated in the state used in the restart or reconnect. (At genesis, this hash should be the - * {@link #ZERO_BLOCK_HASH}.) - * - * @param blockHash the hash of the last block - */ - void initLastBlockHash(@NonNull final Bytes blockHash) { - lastBlockHash = requireNonNull(blockHash); + return stateChangeSubTree; } @Override @@ -443,6 +423,18 @@ public void startRound(@NonNull final Round round, @NonNull final State state) { firstConsensusTimeOfCurrentBlock = round.getConsensusTimestamp(); } + /** + * Initializes the block stream manager after a restart or during reconnect with the hash of the last block + * incorporated in the state used in the restart or reconnect. (At genesis, this hash should be the + * {@link #ZERO_BLOCK_HASH}.) + * + * @param blockHash the hash of the last block + */ + @VisibleForTesting + void initLastBlockHash(@NonNull final Bytes blockHash) { + lastBlockHash = requireNonNull(blockHash); + } + /** * Recovers the contents and proof context of any pending blocks from disk. */ @@ -661,7 +653,8 @@ public boolean endRound(@NonNull final State state, final long roundNum) { inputTreeHasher.intermediateHashingState(), outputTreeHasher.intermediateHashingState(), stateChangesHasher.intermediateHashingState(), - traceDataHasher.intermediateHashingState())); + traceDataHasher.intermediateHashingState(), + asTimestamp(firstConsensusTimeOfCurrentBlock))); ((CommittableWritableStates) writableState).commit(); // Produce one more state change item (i.e. putting the block stream info just constructed into state) From b69dcf84535a7c1ee9c303a9e56d5712d921a339 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Wed, 15 Oct 2025 15:43:31 -0600 Subject: [PATCH 12/63] Protobuf improvements Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index 3dd46ba96f53..9f1d5dc8ef93 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -277,20 +277,22 @@ message BlockItem { /** Identifer for each sub-tree of the block root fixed size tree */ enum SubMerkleTree { ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice - CONSENSUS_HEADER = 1; - INPUT_ITEM = 2; - OUTPUT_ITEM = 3; - STATE_CHANGE_ITEM = 4; - TRACE_ITEM = 5; - FUTURE_1 = 6; // these place holders for future use sub trees, will be renamed if they are used later - FUTURE_2 = 7; - FUTURE_3 = 8; - FUTURE_4 = 9; - FUTURE_5 = 10; - FUTURE_6 = 11; - FUTURE_7 = 12; - FUTURE_8 = 13; - PREVIOUS_ROOT_HASHES = 14; + PREVIOUS_BLOCK_ROOT = 1; + PREVIOUS_ROOTS_TREE = 2; + PREVIOUS_BLOCK_START_STATE = 3; + CONSENSUS_HEADER_ITEMS = 4; + INPUT_ITEMS_TREE = 5; + OUTPUT_ITEMS_TREE = 6; + STATE_CHANGE_ITEMS_TREE = 7; + TRACE_DATA_ITEMS_TREE = 8; + FUTURE_1 = 9; // these place holders for future use sub trees, will be renamed if they are used later + FUTURE_2 = 10; + FUTURE_3 = 11; + FUTURE_4 = 12; + FUTURE_5 = 13; + FUTURE_6 = 14; + FUTURE_7 = 15; + FUTURE_8 = 16; } /** @@ -408,7 +410,7 @@ message RedactedItem { message StreamingTreeSnapshot { /** - * Which of the block merkle sub trees this snapshot is for + * Which of the block merkle sub trees this snapshot represents */ SubMerkleTree type = 1; From 79241180e5f4f48788a4d72026e694e45402e54f Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Wed, 15 Oct 2025 16:27:38 -0600 Subject: [PATCH 13/63] State proof definition Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_proof.proto | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto index fa82307f4f09..e9b85c7bccee 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto @@ -82,7 +82,7 @@ message BlockProof { * This value SHOULD match the block number of the current block, * under normal operation. */ - uint64 block = 1 [deprecated = true]; + uint64 block = 1; /** * A block root hash for the previous block. @@ -227,17 +227,33 @@ message BlockProof { */ ChainOfTrustProof verification_key_proof = 7; - oneof proof { - TssSignedBlockProof signed_block_proof = 8; -// StateProof block_state_proof = 9; - SignedRecordFileProof signed_record_file_proof = 10; - } + /** + * TODO + */ + StateProof state_proof = 8; +} + +/** + * TODO + */ +message StateProof { +// repeated MerklePath paths = 1; + oneof proof { + TssSignedBlockProof signed_block_proof = 2; + SignedRecordFileProof signed_record_file_proof = 3; + } } +/** + * TODO + */ message TssSignedBlockProof { bytes block_signature = 1; } +/** + * TODO + */ message SignedRecordFileProof { /** * A collection of RSA signatures from consensus nodes.
From d36219fc5dfc800d0530bd28c310ffd7fb6667df Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Thu, 16 Oct 2025 00:57:46 -0600 Subject: [PATCH 14/63] Comment out StateProof for now Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_proof.proto | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto index e9b85c7bccee..283a0d25d64a 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto @@ -230,19 +230,19 @@ message BlockProof { /** * TODO */ - StateProof state_proof = 8; +// StateProof state_proof = 8; } /** * TODO */ -message StateProof { -// repeated MerklePath paths = 1; - oneof proof { - TssSignedBlockProof signed_block_proof = 2; - SignedRecordFileProof signed_record_file_proof = 3; - } -} +//message StateProof { +//// repeated MerklePath paths = 1; +// oneof proof { +// TssSignedBlockProof signed_block_proof = 2; +// SignedRecordFileProof signed_record_file_proof = 3; +// } +//} /** * TODO From a8a3f3a123f848bae16a40d1ecd0054e9691ccca Mon Sep 17 00:00:00 2001 From: Zhivko Kelchev Date: Thu, 16 Oct 2025 18:14:19 +0300 Subject: [PATCH 15/63] feat: Send end block request (#21413) Signed-off-by: Zhivko Kelchev --- .../app/blocks/impl/streaming/BlockState.java | 14 +++ .../app/blocks/BlockStreamBuilderTest.java | 1 - .../simulator/SimulatedBlockNodeServer.java | 96 +++++++++++-------- .../bdd/suites/blocknode/BlockNodeSuite.java | 31 ++++++ hiero-dependency-versions/build.gradle.kts | 2 +- 5 files changed, 101 insertions(+), 43 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/streaming/BlockState.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/streaming/BlockState.java index 9b83b820f336..7aaeb8b8eeb0 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/streaming/BlockState.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/streaming/BlockState.java @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.hiero.block.api.BlockEnd; import org.hiero.block.api.BlockItemSet; import org.hiero.block.api.PublishStreamRequest; @@ -292,6 +293,7 @@ public synchronized void processPendingItems(final int batchSize) { final Iterator it = pendingItems.iterator(); boolean forceCreation = false; + boolean sendEndOfBlock = false; while (it.hasNext()) { final BlockItem item = it.next(); blockItems.add(item); @@ -320,6 +322,8 @@ && isPreProofItemReceived(item.stateChangesOrElse(StateChanges.DEFAULT))) { } else if (item.hasBlockProof()) { if (proofItemInfo.packedInRequest(index)) { forceCreation = true; + // send end of block request if the proof is packed + sendEndOfBlock = true; logger.trace("[Block {}] Block proof packed in request #{}", blockNumber, index); } else { logger.warn( @@ -344,6 +348,16 @@ && isPreProofItemReceived(item.stateChangesOrElse(StateChanges.DEFAULT))) { logger.trace("[Block {}] Created new request (index={}, numItems={})", blockNumber, index, blockItems.size()); + if (sendEndOfBlock) { + final var eobRequest = PublishStreamRequest.newBuilder() + .endOfBlock(BlockEnd.newBuilder().blockNumber(blockNumber)) + .build(); + final var eobRequestIndex = requestIdxCtr.getAndIncrement(); + final RequestWrapper rsEnd = new RequestWrapper(eobRequestIndex, eobRequest, new AtomicBoolean(false)); + requestsByIndex.put(eobRequestIndex, rsEnd); + logger.trace("[Block {}] Created new request (index={}, BlockEnd)", blockNumber, eobRequestIndex); + } + if (!pendingItems.isEmpty()) { processPendingItems(batchSize); } diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java index 5485074c0f9d..d6b6e120440e 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java @@ -10,7 +10,6 @@ import static com.hedera.node.app.spi.workflows.HandleContext.TransactionCategory.USER; import static com.hedera.node.app.spi.workflows.record.StreamBuilder.ReversingBehavior.REVERSIBLE; import static com.hedera.node.app.spi.workflows.record.StreamBuilder.SignedTxCustomizer.NOOP_SIGNED_TX_CUSTOMIZER; -import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/hedera/simulator/SimulatedBlockNodeServer.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/hedera/simulator/SimulatedBlockNodeServer.java index 1c8aaa971e9e..41338213b339 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/hedera/simulator/SimulatedBlockNodeServer.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/hedera/simulator/SimulatedBlockNodeServer.java @@ -83,8 +83,8 @@ public class SimulatedBlockNodeServer { // Locks for synchronizing access to block tracking data structures private final ReadWriteLock blockTrackingLock = new ReentrantReadWriteLock(); - // Track all block numbers for which we have received proofs - private final Set blocksWithProofs = ConcurrentHashMap.newKeySet(); + // Track all block numbers for which we have received end block + private final Set blocksWithEndBlock = ConcurrentHashMap.newKeySet(); // Track all block numbers for which we have received headers but not yet proofs private final Set blocksWithHeadersOnly = ConcurrentHashMap.newKeySet(); @@ -250,8 +250,8 @@ public long getLastVerifiedBlockNumber() { public boolean hasReceivedBlock(final long blockNumber) { blockTrackingLock.readLock().lock(); try { - // A block is considered received only if we have its proof - return blocksWithProofs.contains(blockNumber); + // A block is considered received only if we have its end block + return blocksWithEndBlock.contains(blockNumber); } finally { blockTrackingLock.readLock().unlock(); } @@ -267,8 +267,8 @@ public boolean hasReceivedBlock(final long blockNumber) { public Set getReceivedBlockNumbers() { blockTrackingLock.readLock().lock(); try { - // Return only blocks for which we have proofs - return Set.copyOf(blocksWithProofs); + // Return only blocks for which we have the end block + return Set.copyOf(blocksWithEndBlock); } finally { blockTrackingLock.readLock().unlock(); } @@ -382,8 +382,8 @@ public void onNext(final PublishStreamRequest request) { port, replies.hashCode()); - // Requirement 3: Check if block already exists (header AND proof received) - if (blocksWithProofs.contains(blockNumber)) { + // Requirement 3: Check if block already exists (header AND end block received) + if (blocksWithEndBlock.contains(blockNumber)) { log.warn( "Block {} already fully received (header+proof). Sending BlockAcknowledgement to stream {} on port {}.", blockNumber, @@ -461,42 +461,56 @@ public void onNext(final PublishStreamRequest request) { // Continue to the next BlockItem in the request continue; } - - // Mark block as fully received - blocksWithHeadersOnly.remove(blockNumber); - blocksWithProofs.add(blockNumber); - streamingBlocks.remove(blockNumber); // No longer streaming this specific block - - // Update last verified block number atomically - final long newLastVerified = lastVerifiedBlockNumber.updateAndGet( - currentMax -> Math.max(currentMax, blockNumber)); - log.info( - "Block {} fully received (header+proof) on port {} from stream {}. Last verified block updated to: {}", - blockNumber, - port, - replies.hashCode(), - newLastVerified); - - // Requirement 2: Send BlockAcknowledgement to ALL connected pipelines - log.info( - "Broadcasting BlockAcknowledgement for block {} to {} active streams on port {}", - blockNumber, - activeStreams.size(), - port); - for (final Pipeline pipeline : activeStreams) { - if (highLatency) { - // If the simulator is set to be with high latency, delay acknowledgements - // with 1500 ms (assuming CN considers 1000 ms delays as high latency) - Thread.sleep(1500); - } - - buildAndSendBlockAcknowledgement(blockNumber, pipeline); + } + } // End of loop through BlockItems + } else if (request.hasEndOfBlock()) { + final var blockNumber = request.endOfBlockOrThrow().blockNumber(); + if (currentBlockNumber == null + || currentBlockNumber != blockNumber + || !streamingBlocks.containsKey(blockNumber) + || streamingBlocks.get(blockNumber) != replies) { + log.info( + "Received EndBlock for block {} from stream {} on port {}", + blockNumber, + replies.hashCode(), + port); + + // Mark block as fully received + blocksWithHeadersOnly.remove(blockNumber); + blocksWithEndBlock.add(blockNumber); + streamingBlocks.remove(blockNumber); // No longer streaming this specific block + + // Update last verified block number atomically + final long newLastVerified = lastVerifiedBlockNumber.updateAndGet( + currentMax -> Math.max(currentMax, blockNumber)); + log.info( + "Block {} fully received (header+proof) on port {} from stream {}. Last verified block updated to: {}", + blockNumber, + port, + replies.hashCode(), + newLastVerified); + + // Requirement 2: Send BlockAcknowledgement to ALL connected pipelines + log.info( + "Broadcasting BlockAcknowledgement for block {} to {} active streams on port {}", + blockNumber, + activeStreams.size(), + port); + + // send acknowledgment + for (final Pipeline pipeline : activeStreams) { + if (highLatency) { + // If the simulator is set to be with high latency, delay acknowledgements + // with 1500 ms (assuming CN considers 1000 ms delays as high latency) + Thread.sleep(1500); } - // Reset currentBlockNumber for this stream, as it finished sending this block - currentBlockNumber = null; + buildAndSendBlockAcknowledgement(blockNumber, pipeline); } - } // End of loop through BlockItems + + // Reset currentBlockNumber for this stream, as it finished sending this block + currentBlockNumber = null; + } } } catch (InterruptedException e) { log.warn("Interrupted while waiting for BlockAcknowledgement", e); diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java index 8058322cdc60..0b8b58e16e3d 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java @@ -1008,4 +1008,35 @@ private Stream validateHappyPath(int blocksToWait) { assertHgcaaLogDoesNotContain( byNodeId(0), "Block node has exceeded high latency threshold", Duration.ofSeconds(0))); } + + @HapiTest + @HapiBlockNode( + networkSize = 1, + blockNodeConfigs = {@BlockNodeConfig(nodeId = 0, mode = BlockNodeMode.SIMULATOR)}, + subProcessNodeConfigs = { + @SubProcessNodeConfig( + nodeId = 0, + blockNodeIds = {0}, + blockNodePriorities = {0}, + applicationPropertiesOverrides = { + "blockStream.streamMode", "BOTH", + "blockStream.writerMode", "FILE_AND_GRPC" + }) + }) + @Order(13) + final Stream node0SendEndOfBlockHappyPath() { + final AtomicReference timeRef = new AtomicReference<>(); + return hapiTest( + doingContextual(spec -> timeRef.set(Instant.now())), + waitUntilNextBlocks(10).withBackgroundTraffic(true), + // assert no errors + assertHgcaaLogDoesNotContain(byNodeId(0), "ERROR", Duration.ofSeconds(5)), + sourcingContextual(spec -> assertHgcaaLogContainsTimeframe( + byNodeId(0), + timeRef::get, + Duration.ofMinutes(1), + Duration.ofMinutes(1), + // Should send END_OF_BLOCK requests + "Sending request to block node (type=END_OF_BLOCK)"))); + } } diff --git a/hiero-dependency-versions/build.gradle.kts b/hiero-dependency-versions/build.gradle.kts index 7c51ed8f4c65..00af7c0d443d 100644 --- a/hiero-dependency-versions/build.gradle.kts +++ b/hiero-dependency-versions/build.gradle.kts @@ -29,7 +29,7 @@ val log4j = "2.25.0" val mockito = "5.18.0" val pbj = "0.11.15" // ATTENTION: keep in sync with plugin version in 'hapi/hapi/build.gradle.kts' val protobuf = "4.31.1" -val blockNodeProtobufSources = "0.17.1" +val blockNodeProtobufSources = "0.20.0-rc1" val testContainers = "1.21.3" val tuweni = "2.4.2" val webcompare = "2.1.8" From 05928c5ab466d11ca92b4b2f42e3f33b5f18df1f Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Wed, 22 Oct 2025 00:06:41 -0600 Subject: [PATCH 16/63] Temp fix for protobuf cyclic dependency (BlockItem <-> MerkleLeaf) Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_proof.proto | 412 +++++++----------- .../block/stream/chain_of_trust_proof.proto | 67 +++ .../main/proto/block/stream/state_proof.proto | 144 ++++++ .../state/blockstream/merkle_leaf.proto | 14 +- .../state/history/history_types.proto | 2 +- 5 files changed, 388 insertions(+), 251 deletions(-) create mode 100644 hapi/hedera-protobuf-java-api/src/main/proto/block/stream/chain_of_trust_proof.proto create mode 100644 hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto index 283a0d25d64a..5b7fbec48e65 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto @@ -19,6 +19,8 @@ option java_package = "com.hedera.hapi.block.stream.protoc"; option java_multiple_files = true; import "block/stream/record_file_item.proto"; +import "block/stream/chain_of_trust_proof.proto"; +import "block/stream/state_proof.proto"; /** * A cryptographic proof for the "Block Merkle Tree". @@ -74,192 +76,163 @@ import "block/stream/record_file_item.proto"; * leaf. */ message BlockProof { - /** - * The block this proof secures.
- * We provide this because a proof for a future block can be used to prove - * the state of the ledger at that block and the blocks before it.
- *

- * This value SHOULD match the block number of the current block, - * under normal operation. - */ - uint64 block = 1; - - /** - * A block root hash for the previous block. - *

- * This value MUST match the block merkle tree root hash of the previous - * block in the block stream.
- * This value SHALL be empty for the genesis block, and SHALL NOT be empty - * for any other block.
- * Client systems SHOULD optimistically reject any block with a - * `previous_block_proof_hash` that does not match the block hash of the - * previous block and MAY assume the block stream has encountered data - * loss, data corruption, or unauthorized modification. - *

- * The process for computing a block hash is somewhat complex, and involves - * creating a "virtual" merkle tree to obtain the root merkle hash of - * that virtual tree.
- * The merkle tree SHALL have a 4 part structure with 2 internal nodes, - * structured in a strictly binary tree. - *

    - *
  • The merkle tree root SHALL be the parent of both - * internal nodes. - *
      - *
    1. The first "internal" node SHALL be the parent of the - * two "left-most" nodes. - *
        - *
      1. The first leaf MUST be the previous block hash, and is a - * single 48-byte value.
        2. - *
      2. The second leaf MUST be the root of a, strictly binary, - * merkle tree composed of all "input" block items in - * the block.
        - * Input items SHALL be transactions, system transactions, - * and events.
        - * Leaf nodes in this subtree SHALL be ordered in the - * same order that the block items are encountered - * in the stream.
        3. - *
      - *
      2. - *
    2. The second "internal" node SHALL be the parent of the - * two "right-most" nodes. - *
        - *
      1. The third leaf MUST be the root of a, strictly binary, - * merkle tree composed of all "output" block items in - * the block.
        - * Output items SHALL be transaction result, transaction - * output, and state changes.
        - * Leaf nodes in this subtree SHALL be ordered in the - * same order that the block items are encountered - * in the stream.
        2. - *
      2. The fourth leaf MUST be the merkle tree root hash for - * network state at the start of the block, and is a single - * 48-byte value.
        3. - *
      - *
      3. - *
    - *
    • - *
  • The block hash SHALL be the SHA-384 hash calculated for the root - * of this merkle tree.
    • - *
- */ - bytes previous_block_root_hash = 2 [deprecated = true]; - - /** - * A merkle root hash of the network state.
- * This is present to support validation of this block proof by clients - * that do not maintain a full copy of the network state. - *

- * This MUST contain a hash of the "state" merkle tree root at the start - * of the current block (which this block proof verifies).
- * State processing clients SHOULD calculate the state root hash - * independently and SHOULD NOT rely on this value.
- * State processing clients MUST validate the application of state changes - * for a block using the value present in the Block Proof of the - * _following_ block. - * Compliant consensus nodes MUST produce an "empty" block (containing - * only `BlockHeader` and `BlockProof` as the last block prior to a - * network "freeze" to ensure the final state hash is incorporated into - * the Block Stream correctly. - * Stateless (non-state-processing) clients MUST use this value to - * construct the block merkle tree. - */ - bytes start_of_block_state_root_hash = 3 [deprecated = true]; - - /** - * A TSS signature for one block.
- * This is a single signature representing the collection of partial - * signatures from nodes holding strictly greater than 2/3 of the - * current network "weight" in aggregate. The signature is produced by - * cryptographic "aggregation" of the partial signatures to produce a - * single signature that can be verified with the network public key, - * but could not be produced by fewer nodes than required to meet the - * threshold for network stake "weight". - *

- * This message MUST make use of a threshold signature scheme like `BLS` - * which provides the necessary cryptographic guarantees.
- * This signature SHALL use a TSS signature to provide a single signature - * that represents the consensus signature of consensus nodes.
- * The exact subset of nodes that signed SHALL neither be known nor - * tracked, but it SHALL be cryptographically verifiable that the - * threshold was met if the signature itself can be validated with - * the network public key (a.k.a `LedgerID`). - */ - bytes block_signature = 4 [deprecated = true]; + /** + * The block this proof secures.
+ * We provide this because a proof for a future block can be used to prove + * the state of the ledger at that block and the blocks before it.
+ *

+ * This value SHOULD match the block number of the current block, + * under normal operation. + */ + uint64 block = 1; - /** - * A set of hash values along with ordering information.
- * This list of hash values form the set of sibling hash values needed to - * correctly reconstruct the parent hash, and all hash values "above" that - * hash in the merkle tree. - *

- * A Block proof can be constructed by combining the sibling hashes for - * a previous block hash and sibling hashes for each entry "above" that - * node in the merkle tree of a block proof that incorporates that previous - * block hash. This form of block proof may be used to prove a chain of - * blocks when one or more older blocks is missing the original block - * proof that signed the block's merkle root directly. - *

- * This list MUST be ordered from the sibling of the node that contains - * this block's root node hash, and continues up the merkle tree to the - * root hash of the signed block proof. - *

- * If this block proof has a "direct" signature, then this list MUST be - * empty.
- * If this list is not empty, then this block proof MUST be verified by - * first constructing the "block" merkle tree and computing the root hash - * of that tree, then combining that hash with the values in this list, - * paying attention to the first/second sibling ordering, until the root - * merkle hash is produced from the last pair of sibling hashes. That - * "secondary" root hash MUST then be verified using - * the value of `block_signature`. - */ - repeated MerkleSiblingHash sibling_hashes = 5 [deprecated = true]; + /** + * A block root hash for the previous block. + *

+ * This value MUST match the block merkle tree root hash of the previous + * block in the block stream.
+ * This value SHALL be empty for the genesis block, and SHALL NOT be empty + * for any other block.
+ * Client systems SHOULD optimistically reject any block with a + * `previous_block_proof_hash` that does not match the block hash of the + * previous block and MAY assume the block stream has encountered data + * loss, data corruption, or unauthorized modification. + *

+ * The process for computing a block hash is somewhat complex, and involves + * creating a "virtual" merkle tree to obtain the root merkle hash of + * that virtual tree.
+ * The merkle tree SHALL have a 4 part structure with 2 internal nodes, + * structured in a strictly binary tree. + *

    + *
  • The merkle tree root SHALL be the parent of both + * internal nodes. + *
      + *
    1. The first "internal" node SHALL be the parent of the + * two "left-most" nodes. + *
        + *
      1. The first leaf MUST be the previous block hash, and is a + * single 48-byte value.
        2. + *
      2. The second leaf MUST be the root of a, strictly binary, + * merkle tree composed of all "input" block items in + * the block.
        + * Input items SHALL be transactions, system transactions, + * and events.
        + * Leaf nodes in this subtree SHALL be ordered in the + * same order that the block items are encountered + * in the stream.
        3. + *
      + *
      2. + *
    2. The second "internal" node SHALL be the parent of the + * two "right-most" nodes. + *
        + *
      1. The third leaf MUST be the root of a, strictly binary, + * merkle tree composed of all "output" block items in + * the block.
        + * Output items SHALL be transaction result, transaction + * output, and state changes.
        + * Leaf nodes in this subtree SHALL be ordered in the + * same order that the block items are encountered + * in the stream.
        2. + *
      2. The fourth leaf MUST be the merkle tree root hash for + * network state at the start of the block, and is a single + * 48-byte value.
        3. + *
      + *
      3. + *
    + *
    • + *
  • The block hash SHALL be the SHA-384 hash calculated for the root + * of this merkle tree.
    • + *
+ */ + bytes previous_block_root_hash = 2 [deprecated = true]; - /** - * The hinTS key that this signature verifies under; a stream consumer should - * only use this key after first checking the chain of trust proof. - */ - bytes verification_key = 6; + /** + * A merkle root hash of the network state.
+ * This is present to support validation of this block proof by clients + * that do not maintain a full copy of the network state. + *

+ * This MUST contain a hash of the "state" merkle tree root at the start + * of the current block (which this block proof verifies).
+ * State processing clients SHOULD calculate the state root hash + * independently and SHOULD NOT rely on this value.
+ * State processing clients MUST validate the application of state changes + * for a block using the value present in the Block Proof of the + * _following_ block. + * Compliant consensus nodes MUST produce an "empty" block (containing + * only `BlockHeader` and `BlockProof` as the last block prior to a + * network "freeze" to ensure the final state hash is incorporated into + * the Block Stream correctly. + * Stateless (non-state-processing) clients MUST use this value to + * construct the block merkle tree. + */ + bytes start_of_block_state_root_hash = 3 [deprecated = true]; - /** - * Proof the hinTS verification key is in the chain of trust extending - * from the network's ledger id. - */ - ChainOfTrustProof verification_key_proof = 7; + /** + * A TSS signature for one block.
+ * This is a single signature representing the collection of partial + * signatures from nodes holding strictly greater than 2/3 of the + * current network "weight" in aggregate. The signature is produced by + * cryptographic "aggregation" of the partial signatures to produce a + * single signature that can be verified with the network public key, + * but could not be produced by fewer nodes than required to meet the + * threshold for network stake "weight". + *

+ * This message MUST make use of a threshold signature scheme like `BLS` + * which provides the necessary cryptographic guarantees.
+ * This signature SHALL use a TSS signature to provide a single signature + * that represents the consensus signature of consensus nodes.
+ * The exact subset of nodes that signed SHALL neither be known nor + * tracked, but it SHALL be cryptographically verifiable that the + * threshold was met if the signature itself can be validated with + * the network public key (a.k.a `LedgerID`). + */ + bytes block_signature = 4 [deprecated = true]; - /** - * TODO - */ -// StateProof state_proof = 8; -} + /** + * A set of hash values along with ordering information.
+ * This list of hash values form the set of sibling hash values needed to + * correctly reconstruct the parent hash, and all hash values "above" that + * hash in the merkle tree. + *

+ * A Block proof can be constructed by combining the sibling hashes for + * a previous block hash and sibling hashes for each entry "above" that + * node in the merkle tree of a block proof that incorporates that previous + * block hash. This form of block proof may be used to prove a chain of + * blocks when one or more older blocks is missing the original block + * proof that signed the block's merkle root directly. + *

+ * This list MUST be ordered from the sibling of the node that contains + * this block's root node hash, and continues up the merkle tree to the + * root hash of the signed block proof. + *

+ * If this block proof has a "direct" signature, then this list MUST be + * empty.
+ * If this list is not empty, then this block proof MUST be verified by + * first constructing the "block" merkle tree and computing the root hash + * of that tree, then combining that hash with the values in this list, + * paying attention to the first/second sibling ordering, until the root + * merkle hash is produced from the last pair of sibling hashes. That + * "secondary" root hash MUST then be verified using + * the value of `block_signature`. + */ + repeated MerkleSiblingHash sibling_hashes = 5 [deprecated = true]; -/** - * TODO - */ -//message StateProof { -//// repeated MerklePath paths = 1; -// oneof proof { -// TssSignedBlockProof signed_block_proof = 2; -// SignedRecordFileProof signed_record_file_proof = 3; -// } -//} + /** + * The hinTS key that this signature verifies under; a stream consumer should + * only use this key after first checking the chain of trust proof. + */ + bytes verification_key = 6; -/** - * TODO - */ -message TssSignedBlockProof { - bytes block_signature = 1; -} + /** + * Proof the hinTS verification key is in the chain of trust extending + * from the network's ledger id. + */ + ChainOfTrustProof verification_key_proof = 7; -/** - * TODO - */ -message SignedRecordFileProof { - /** - * A collection of RSA signatures from consensus nodes.
- * These signatures validate the hash of the record_file_contents field. - */ - repeated RecordFileSignature record_file_signatures = 1; + /** + * TODO + */ + com.hedera.hapi.block.stream.StateProof state_proof = 8; } /** @@ -273,72 +246,23 @@ message SignedRecordFileProof { * a node of interest up to the root of the tree. */ message MerkleSiblingHash { - /** - * A flag for the position of this sibling. - *

- * If this is set then this sibling MUST be the first hash in the pair of - * sibling hashes of a binary merkle tree.
- * If this is unset, then this sibling MUST be the second hash in the pair - * of sibling hashes of a binary merkle tree. - */ - bool is_first = 1; - - /** - * A byte array of a sibling hash.
- * This is the hash for the sibling at this point in the merkle tree. - *

- * The algorithm for this hash SHALL match the algorithm for the block that - * contains this sibling.
- * This SHALL contain the raw (e.g.) 384 bits (48 bytes) of the hash value. - */ - bytes sibling_hash = 2; -} - -/** - * Proof that some data belongs to the network's chain of trust, starting from - * the genesis network whose address book hash formed the ledger id. (In the - * current system, this data is always a hinTS verification key.) - */ -message ChainOfTrustProof { - oneof proof { - /** - * If there is not yet a SNARK proving the chain of trust from ledger id to - * the hinTS verification key, the explicit list of Schnorr signatures on - * the concatenation of the ledger id and genesis hinTS verification key - * that serve as witnesses for the SNARK prover algorithm. - */ - NodeSignatures node_signatures = 1; - /** - * If known, a ZK-compressed SNARK proof proving the chain of trust from - * the ledger id to this hinTS verification key. - */ - bytes wraps_proof = 2; - } -} - -/** - * A list of valid node signatures on some data assumed known from the context - * of the message, ordered by node id. - *

- * Can be used to prove the genesis hinTS verification key in a block proof; but - * not succinct and not recursive; hence in normal operations with TSS, used only - * until the first recursive proof is available. - */ -message NodeSignatures { - repeated NodeSignature node_signatures = 1; -} - -/** - * A signature from a node on some data assumed known from the context of the - * message. - */ -message NodeSignature { /** - * The node id of the signer. + * A flag for the position of this sibling. + *

+ * If this is set then this sibling MUST be the first hash in the pair of + * sibling hashes of a binary merkle tree.
+ * If this is unset, then this sibling MUST be the second hash in the pair + * of sibling hashes of a binary merkle tree. */ - uint64 node_id = 1; + bool is_first = 1; + /** - * The signature. + * A byte array of a sibling hash.
+ * This is the hash for the sibling at this point in the merkle tree. + *

+ * The algorithm for this hash SHALL match the algorithm for the block that + * contains this sibling.
+ * This SHALL contain the raw (e.g.) 384 bits (48 bytes) of the hash value. */ - bytes signature = 2; -} + bytes sibling_hash = 2; +} \ No newline at end of file diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/chain_of_trust_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/chain_of_trust_proof.proto new file mode 100644 index 000000000000..837d8802f339 --- /dev/null +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/chain_of_trust_proof.proto @@ -0,0 +1,67 @@ +/** + * TODO + * + * ### Keywords + * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + * document are to be interpreted as described in + * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in + * [RFC8174](https://www.ietf.org/rfc/rfc8174). + */ +syntax = "proto3"; + +package com.hedera.hapi.block.stream; + +// SPDX-License-Identifier: Apache-2.0 +option java_package = "com.hedera.hapi.block.stream.protoc"; +// <<>> This comment is special code for setting PBJ Compiler java package +option java_multiple_files = true; + +/** + * Proof that some data belongs to the network's chain of trust, starting from + * the genesis network whose address book hash formed the ledger id. (In the + * current system, this data is always a hinTS verification key.) + */ +message ChainOfTrustProof { + oneof proof { + /** + * If there is not yet a SNARK proving the chain of trust from ledger id to + * the hinTS verification key, the explicit list of Schnorr signatures on + * the concatenation of the ledger id and genesis hinTS verification key + * that serve as witnesses for the SNARK prover algorithm. + */ + NodeSignatures node_signatures = 1; + /** + * If known, a ZK-compressed SNARK proof proving the chain of trust from + * the ledger id to this hinTS verification key. + */ + bytes wraps_proof = 2; + } +} + +/** + * A list of valid node signatures on some data assumed known from the context + * of the message, ordered by node id. + *

+ * Can be used to prove the genesis hinTS verification key in a block proof; but + * not succinct and not recursive; hence in normal operations with TSS, used only + * until the first recursive proof is available. + */ +message NodeSignatures { + repeated NodeSignature node_signatures = 1; +} + +/** + * A signature from a node on some data assumed known from the context of the + * message. + */ +message NodeSignature { + /** + * The node id of the signer. + */ + uint64 node_id = 1; + /** + * The signature. + */ + bytes signature = 2; +} \ No newline at end of file diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto new file mode 100644 index 000000000000..6ab1ed2fd551 --- /dev/null +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto @@ -0,0 +1,144 @@ +/** + * TODO + * + * ### Keywords + * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + * document are to be interpreted as described in + * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in + * [RFC8174](https://www.ietf.org/rfc/rfc8174). + */ +syntax = "proto3"; + +package com.hedera.hapi.block.stream; + +// SPDX-License-Identifier: Apache-2.0 +option java_package = "com.hedera.hapi.block.stream.protoc"; +// <<>> This comment is special code for setting PBJ Compiler java package +option java_multiple_files = true; + +import "services/timestamp.proto"; +import "block/stream/record_file_item.proto"; +import "services/state/blockstream/merkle_leaf.proto"; + +/** + * TODO + */ +message StateProof { + + /** + * TODO + */ + repeated MerklePath paths = 1; + + oneof proof { + /** + * TODO + */ + TssSignedBlockProof signed_block_proof = 2; + + /** + * TODO + */ + SignedRecordFileProof signed_record_file_proof = 3; + } +} + +/** + * TODO + */ +message TssSignedBlockProof { + + /** + * A TSS signature for one block.
+ * This is a single signature representing the collection of partial + * signatures from nodes holding strictly greater than 2/3 of the + * current network "weight" in aggregate. The signature is produced by + * cryptographic "aggregation" of the partial signatures to produce a + * single signature that can be verified with the network public key, + * but could not be produced by fewer nodes than required to meet the + * threshold for network stake "weight". + *

+ * This message MUST make use of a threshold signature scheme like `BLS` + * which provides the necessary cryptographic guarantees.
+ * This signature SHALL use a TSS signature to provide a single signature + * that represents the consensus signature of consensus nodes.
+ * The exact subset of nodes that signed SHALL neither be known nor + * tracked, but it SHALL be cryptographically verifiable that the + * threshold was met if the signature itself can be validated with + * the network public key (a.k.a `LedgerID`). + */ + bytes block_signature = 1; +} + +/** + * TODO + */ +message SignedRecordFileProof { + + /** + * A collection of RSA signatures from consensus nodes.
+ * These signatures validate the hash of the record_file_contents field. + */ + repeated RecordFileSignature record_file_signatures = 1; +} + +/** + * A path from a node in a Merkle tree to the root of that tree. + * + * MerklePath represents a section of a merkle path from a leaf node to merkle + * tree root. There are 3 kinds of merkle paths. + * 1. Merkle paths starting from a leaf. For these the `leaf` field is set. + * 2. Merkle paths starting from an internal node hash like for proving a + * block root hash. For these the `hash` field is set. + * 3. Merkle paths in the middle between another merkle path and the root. For + * these neither `leaf` for `hash` fields are set. + * Merkle paths can only include sibling nodes for binary internal nodes. If + * there is a unary node in the path a new MerklePath must be started. + */ +message MerklePath { + + oneof content { + /** + * Optional leaf, if this path starts from a leaf + */ + com.hedera.hapi.node.state.blockstream.MerkleLeaf leaf = 1; + + /** + * Optional hash content for a path with no leaf that hashes paths below + * it. Needed for cases like proving a block root hash from another block. + */ + bytes hash = 2; + } + + /** + * Array of sibling nodes ordered from bottom of tree to top + */ + repeated SiblingNode siblings = 3; + + /** + * The next parent path of this path going up the tree. Expressed as an index + * into the array of MerklePaths in the StatePoof. For example 0 being first + * in list etc. If this is the root path then the value is UINT32_MAX + * (this is `-1` in Java; 0xFFFFFFFF). + */ + uint32 nextPathIndex = 4; +} + +/** + * Sibling Node, this represents the hash of a sibling node in a MerklePath + */ +message SiblingNode { + /** + * True when this sibling is on the left of the merkle path, False if on + * the right side of the merkle path + */ + bool is_left = 1; + + /** + * The hash of the sibling node that can be combined with the merkle path + * computed hashes as the proof is validated. + */ + bytes hash = 2; +} + diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto index edf6223ccd3d..6d1bf7d9ed41 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto @@ -1,6 +1,6 @@ /** * Merkle Leaf - * TODO!! + * TODO * * ### Keywords * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", @@ -16,7 +16,7 @@ package com.hedera.hapi.node.state.blockstream; // SPDX-License-Identifier: Apache-2.0 import "services/timestamp.proto"; import "platform/state/virtual_map_state.proto"; -import "block/stream/block_item.proto"; +//import "block/stream/block_item.proto"; option java_package = "com.hedera.hapi.block.stream.protoc"; // <<>> This comment is special code for setting PBJ Compiler java package @@ -33,14 +33,16 @@ message MerkleLeaf { proto.Timestamp block_consensus_timestamp = 1; /** - * TODO + * TODO – SHOULD BE BlockItem, not bytes */ - com.hedera.hapi.block.stream.BlockItem block_item = 2; + // com.hedera.hapi.block.stream.BlockItem block_item = 2; + bytes block_item = 2; /** - * TODO + * TODO – SHOULD BE StateItem, not bytes */ - com.hedera.hapi.platform.state.StateItem state_item = 3; + // com.hedera.hapi.platform.state.StateItem state_item = 3; + bytes state_item = 3; } } diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/history/history_types.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/history/history_types.proto index dbbf587215e8..49d0a7f707b0 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/history/history_types.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/history/history_types.proto @@ -4,7 +4,7 @@ package com.hedera.hapi.node.state.history; // SPDX-License-Identifier: Apache-2.0 import "services/timestamp.proto"; -import "block/stream/block_proof.proto"; +import "block/stream/chain_of_trust_proof.proto"; option java_package = "com.hederahashgraph.api.proto.java"; // <<>> This comment is special code for setting PBJ Compiler java package From c45d2a7bba877171d7cda66feae831c6382b9d6a Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Fri, 26 Sep 2025 13:29:04 -0600 Subject: [PATCH 17/63] Add initial protobufs for new block merkle tree structure Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 105 ++++++++++++++++++ .../block/stream/output/block_footer.proto | 41 +++++++ .../block/stream/output/block_proof2.proto | 62 +++++++++++ .../state/blockstream/merkle_leaf.proto | 46 ++++++++ 4 files changed, 254 insertions(+) create mode 100644 hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto create mode 100644 hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto create mode 100644 hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index a5db0820b8cd..b6b91c2a9d93 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -292,3 +292,108 @@ message FilteredItemHash { */ uint64 filtered_path = 3; } + +/** Identifer for each sub-tree of the block root fixed size tree */ +enum SubMerkleTree { + ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice + CONSENSUS_HEADER = 1; + INPUT_ITEM = 2; + OUTPUT_ITEM = 3; + STATE_CHANGE_ITEM = 4; + TRACE_ITEM = 5; + FUTURE_1 = 6; // these place holders for future use sub trees, will be renamed if they are used later + FUTURE_2 = 7; + FUTURE_3 = 8; + FUTURE_4 = 9; + FUTURE_5 = 10; + FUTURE_6 = 11; + FUTURE_7 = 12; + FUTURE_8 = 13; +} + +/** + * Verification data for an item filtered from the stream. + * + * Items of this type SHALL NOT be present in the full (unfiltered) block + * stream.
+ * Items of this type SHALL replace any item removed from a partial (filtered) + * block stream.
+ * Presence of `filtered_item` entries SHALL NOT prevent verification + * of a block, but MAY preclude verification or reconstruction + * of consensus state.
+ */ +message FilteredSingleItem { + /** + * A hash of an item filtered from the stream. + *

+ * The hash algorithm used MUST match the hash algorithm specified in + * the block header for the containing block.
+ * This field is REQUIRED. + */ + bytes item_hash = 1; + + /** + * This tells you which of the block merkle sub trees to add the item hash into + *

+ * This REQUIRED field SHALL describe the type of filtered item + */ + SubMerkleTree tree = 2; +} + +/** + * Filtered Block Item representing a complete block sub merkle tree that has been + * filtered out and replaced by a hash. + */ +message FilteredMerkleSubTree { + /** + * Root hash of a sub-merkle tree + */ + bytes subtree_root_hash = 1; + + /** + * This tells you which of the block merkle sub trees the hash is the root for + */ + SubMerkleTree tree = 2; + + /** + * The number of leaves filtered by this FilteredMerkleSubTree. + */ + uint32 filtered_leaf_count = 3; +} + +/** + * Verification data for an item redacted from the stream. + * + * Presence of `redacted_item` entries SHALL NOT prevent verification + * of a block.
+ */ +message RedactedItem { + /** + * A hash of an item redacted from the stream. + *

+ * The hash algorithm used MUST match the hash algorithm specified in + * the block header for the containing block.
+ * This field is REQUIRED. + */ + bytes item_hash = 1; + + /** + * When thise redacted item is a SignedTransaction, this value is the hash of that SignedTransaction + * directly, without the BlockItem wrapper. This is needed for event reconstruction. The + * signed_transaction_hash will only be set for event transactions, synthetic transactions will have + * empty value. + *

+ * The hash algorithm used MUST match the hash algorithm specified in + * the block header for the containing block.
+ * This value SHALL NOT be provided if the original item MUST NOT be included in an + * event hash. + */ + bytes signed_transaction_hash = 2; + + /** + * This tells you which of the block merkle sub trees to add the item hash into + *

+ * This REQUIRED field SHALL describe the type of filtered item + */ + SubMerkleTree tree = 3; +} diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto new file mode 100644 index 000000000000..60550e1f482a --- /dev/null +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto @@ -0,0 +1,41 @@ +/** + * # Block Footer + * TODO + * + * ### Keywords + * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + * document are to be interpreted as described in + * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in + * [RFC8174](https://www.ietf.org/rfc/rfc8174). + */ +syntax = "proto3"; + +package com.hedera.hapi.block.stream.output; + +// SPDX-License-Identifier: Apache-2.0 +option java_package = "com.hedera.hapi.block.stream.output.protoc"; +// <<>> This comment is special code for setting PBJ Compiler java package +option java_multiple_files = true; + +/** + * A collection of hashes of sub parts of the blocks top fixed merkle tree that are needed to compute the + * blocks root hash. These are the hashes of the first 3 nodes across the bottom of the block fixed merkle + * tree in field order. + */ +message BlockFooter { + + /** The root hash of the block, for the previous block to the one this footer belongs to. + */ + bytes previous_block_root_hash = 1; + + /** + * The root hash of a merkle tree containg the root hashes of all block from block zero up to but not + * including this current block. + */ + bytes root_hash_of_all_block_hashes_tree = 2; + + /** The root hash of the state merkle tree for the version of state at the begining of the current block + */ + bytes start_of_block_state_root_hash = 3; +} diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto new file mode 100644 index 000000000000..3d6fd716e47a --- /dev/null +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto @@ -0,0 +1,62 @@ +/** + * # Block Proof (v2) + * TODO + * Going to use a new block proof definition for now, replacing pieces until the original Block Proof + * definition isn't used. Then we'll remove the original. + * + * ### Keywords + * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + * document are to be interpreted as described in + * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in + * [RFC8174](https://www.ietf.org/rfc/rfc8174). + */ +syntax = "proto3"; + +package com.hedera.hapi.block.stream.output; + +// SPDX-License-Identifier: Apache-2.0 +option java_package = "com.hedera.hapi.block.stream.output.protoc"; +// <<>> This comment is special code for setting PBJ Compiler java package +option java_multiple_files = true; + +import "block/stream/record_file_item.proto"; + +/** + * TODO + */ +message BlockFooter { + + /** + * TODO + */ + bytes previous_block_root_hash = 1; + + /** + * TODO + */ + bytes root_hash_of_all_block_hashes_tree = 2; + + /** + * TODO + */ + bytes start_of_block_state_root_hash = 3; +} + +message TssSignedBlockProof { + bytes block_signature = 4; + + // TODO: probably will be deleted? + oneof verification_reference { + uint64 scheme_id = 6; + bytes verification_key = 7; // extracted from ledger ID? + } +} + +message SignedRecordFileProof { + /** + * A collection of RSA signatures from consensus nodes.
+ * These signatures validate the hash of the record_file_contents field. + */ + repeated com.hedera.hapi.block.stream.RecordFileSignature record_file_signatures = 1; +} diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto new file mode 100644 index 000000000000..edf6223ccd3d --- /dev/null +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto @@ -0,0 +1,46 @@ +/** + * Merkle Leaf + * TODO!! + * + * ### Keywords + * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + * document are to be interpreted as described in + * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in + * [RFC8174](https://www.ietf.org/rfc/rfc8174). + */ +syntax = "proto3"; + +package com.hedera.hapi.node.state.blockstream; + +// SPDX-License-Identifier: Apache-2.0 +import "services/timestamp.proto"; +import "platform/state/virtual_map_state.proto"; +import "block/stream/block_item.proto"; + +option java_package = "com.hedera.hapi.block.stream.protoc"; +// <<>> This comment is special code for setting PBJ Compiler java package +option java_multiple_files = true; + +/** + * TODO + */ +message MerkleLeaf { + oneof content { + /** + * TODO + */ + proto.Timestamp block_consensus_timestamp = 1; + + /** + * TODO + */ + com.hedera.hapi.block.stream.BlockItem block_item = 2; + + /** + * TODO + */ + com.hedera.hapi.platform.state.StateItem state_item = 3; + } +} + From d0200d2e9b69981a504d06acccaeaf3e3643a004 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Fri, 26 Sep 2025 13:29:38 -0600 Subject: [PATCH 18/63] Add algorithm for streaming merkle tree Signed-off-by: Matt Hess --- .../impl/IncrementalStreamingHasher.java | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java new file mode 100644 index 000000000000..10addc43cb03 --- /dev/null +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.node.app.blocks.impl; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.LinkedList; +import java.util.List; + +/** + * A class that computes a Merkle tree root hash in a streaming fashion. It supports adding leaves one by one and + * computes the root hash without storing the entire tree in memory. It uses SHA-384 as the hashing algorithm and + * follows the prefixing scheme for leaves and internal nodes. + * + *

This is not thread safe, it is assumed use by single thread.

+ */ +public class IncrementalStreamingHasher { + /** Prefix byte for hash contents for leaf nodes. */ + private static final byte[] LEAF_PREFIX = new byte[] {0}; + /** Prefix byte for hash contents for internal nodes. */ + private static final byte[] INTERNAL_NODE_PREFIX = new byte[] {2}; + /** The hashing algorithm used for computing the hashes. */ + private final MessageDigest digest; + /** A list to store intermediate hashes as we build the tree. */ + private final LinkedList hashList = new LinkedList<>(); + /** The count of leaves in the tree. */ + private long leafCount = 0; + + /** Create a new StreamingHasher with an empty state. */ + public IncrementalStreamingHasher() { + try { + digest = MessageDigest.getInstance("SHA-384"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + /** + * Create a StreamingHasher with an existing intermediate hashing state. + * This allows resuming hashing from a previous state. + * + * @param intermediateHashingState the intermediate hashing state + */ + public IncrementalStreamingHasher(List intermediateHashingState) { + this(); + this.hashList.addAll(intermediateHashingState); + } + + /** + * Add a new leaf to the Merkle tree. + * + * @param data the data for the new leaf + */ + public void addLeaf(byte[] data) { + final long i = leafCount; + final byte[] e = hashLeaf(data); + hashList.add(e); + for (long n=i; (n & 1L) == 1; n >>= 1) { + final byte[] y = hashList.removeLast(); + final byte[] x = hashList.removeLast(); + hashList.add(hashInternalNode(x, y)); + } + leafCount ++; + } + + /** + * Compute the Merkle tree root hash from the current state. This does not modify the internal state, so can be + * called at any time and more leaves can be added afterward. + * + * @return the Merkle tree root hash + */ + public byte[] computeRootHash() { + byte[] merkleRootHash = hashList.getLast(); + for (int i = hashList.size() - 2; i >= 0; i--) { + merkleRootHash = hashInternalNode(hashList.get(i), merkleRootHash); + } + return merkleRootHash; + } + + /** + * Get the current intermediate hashing state. This can be used to save the state and resume hashing later. + * + * @return the intermediate hashing state + */ + public List intermediateHashingState() { + return hashList; + } + + /** + * Get the number of leaves added to the tree so far. + * + * @return the number of leaves + */ + public long leafCount() { + return leafCount; + } + + /** + * Hash a leaf node with the appropriate prefix. + * + * @param leafData the data of the leaf + * @return the hash of the leaf node + */ + private byte[] hashLeaf(final byte[] leafData) { + digest.update(LEAF_PREFIX); + return digest.digest(leafData); + } + + /** + * Hash an internal node by combining the hashes of its two children with the appropriate prefix. + * + * @param firstChild the hash of the first child + * @param secondChild the hash of the second child + * @return the hash of the internal node + */ + private byte[] hashInternalNode(final byte[] firstChild, final byte[] secondChild) { + digest.update(INTERNAL_NODE_PREFIX); + digest.update(firstChild); + return digest.digest(secondChild); + } +} From c98d4d26c69419f8e7334e6bb8067d80572db100 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Fri, 26 Sep 2025 13:51:36 -0600 Subject: [PATCH 19/63] Update BlockItem with new definition Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 94 +++---------------- .../block/stream/output/block_proof2.proto | 62 ------------ .../app/blocks/impl/BlockStreamBuilder.java | 16 +--- .../blocks/impl/BlockStreamManagerImpl.java | 8 +- .../app/blocks/BlockStreamBuilderTest.java | 43 +++++---- .../impl/BlockStreamManagerImplTest.java | 4 +- 6 files changed, 47 insertions(+), 180 deletions(-) delete mode 100644 hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index b6b91c2a9d93..f7ca563c7a4b 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -37,6 +37,7 @@ import "block/stream/output/state_changes.proto"; import "block/stream/output/transaction_output.proto"; import "block/stream/output/transaction_result.proto"; import "block/stream/trace/trace_data.proto"; +import "block/stream/output/block_footer.proto"; /** * A single item within a block stream. @@ -68,9 +69,11 @@ import "block/stream/trace/trace_data.proto"; * transaction_result * (optional) transaction_output * (optional) repeated state_changes + * (optional) filtered_single_item * } + * block_footer + * repeated block_proof * } - * state_proof * ``` * * A filtered stream may exclude some items above, depending on filter @@ -115,6 +118,7 @@ import "block/stream/trace/trace_data.proto"; * - The "BridgeTransform" field is 24 (24 modulo 10 is 4, so it is Trace Data). * * #### Initial Field assignment to subtree categories. + * TODO: REDEFINE * - Consensus Headers * - `event_header` * - `round_header` @@ -203,96 +207,22 @@ message BlockItem { com.hedera.hapi.block.stream.output.StateChanges state_changes = 7; /** - * Verification data for an item filtered from the stream.
- * This is a hash for a merkle tree node where the contents of that - * part of the merkle tree have been removed from this stream. - *

- * Items of this type SHALL NOT be present in the full (unfiltered) - * block stream.
- * Items of this type SHALL replace any item removed from a partial - * (filtered) block stream.
- * Presence of `filtered_item` entries SHALL NOT prevent verification - * of a block, but MAY preclude verification or reconstruction of - * consensus state.
+ * TODO */ - FilteredItemHash filtered_item_hash = 8; + FilteredSingleItem filtered_single_item = 8; /** - * A signed block proof.
- * The signed merkle proof for this block. This will validate - * a "virtual" merkle tree containing the previous block "virtual" - * root, an "input" subtree, an "output" subtree, and - * a "state changes" subtree. - *

- * This item is not part of the block stream hash chain/tree, and - * MUST follow after the end of a block. + * TODO */ - BlockProof block_proof = 9; + com.hedera.hapi.block.stream.output.BlockFooter block_footer = 9; /** - * A record file and associated data. - *

- * This MUST contain a single Record file, associated Sidecar files, - * and data from related Signature files. - * If this item is present, special treatment is - * REQUIRED for this block. - *

    - *
  • The block SHALL NOT have a `BlockHeader`.
    • - *
  • The block SHALL NOT have a `BlockProof`.
    • - *
  • The block SHALL contain _exactly one_ `RecordFileItem`.
    • - *
  • The block SHALL NOT contain any item other than a - * `RecordFileItem`.
    • - *
  • The content of the `RecordFileItem` MUST be validated using - * the signature data and content provided within according to - * the process used for Record Files prior to the creation - * of Block Stream.
    • - *
+ * TODO */ - RecordFileItem record_file = 10; - - /** - * A trace data. - *

- * Any informational trace data MAY be described by - * stream items of this type.
- */ - com.hedera.hapi.block.stream.trace.TraceData trace_data = 11; + BlockProof block_proof = 10; } } -/** - * Verification data for an item filtered from the stream. - * - * Items of this type SHALL NOT be present in the full (unfiltered) block - * stream.
- * Items of this type SHALL replace any item removed from a partial (filtered) - * block stream.
- * Presence of `filtered_item` entries SHALL NOT prevent verification - * of a block, but MAY preclude verification or reconstruction - * of consensus state.
- */ -message FilteredItemHash { - /** - * A hash of an item filtered from the stream. - *

- * The hash algorithm used MUST match the hash algorithm specified in - * the block header for the containing block.
- * This field is REQUIRED. - */ - bytes item_hash = 1; - - /** - * A record of the merkle path to the item that was filtered - * from the stream.
- * This path begins at the root of the block proof merkle tree. - *

- * This REQUIRED field SHALL describe the full path in the virtual - * merkle tree constructed for the block proof that contained the - * item filtered from the stream. - */ - uint64 filtered_path = 3; -} - /** Identifer for each sub-tree of the block root fixed size tree */ enum SubMerkleTree { ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice @@ -378,7 +308,7 @@ message RedactedItem { bytes item_hash = 1; /** - * When thise redacted item is a SignedTransaction, this value is the hash of that SignedTransaction + * When this redacted item is a SignedTransaction, this value is the hash of that SignedTransaction * directly, without the BlockItem wrapper. This is needed for event reconstruction. The * signed_transaction_hash will only be set for event transactions, synthetic transactions will have * empty value. diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto deleted file mode 100644 index 3d6fd716e47a..000000000000 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_proof2.proto +++ /dev/null @@ -1,62 +0,0 @@ -/** - * # Block Proof (v2) - * TODO - * Going to use a new block proof definition for now, replacing pieces until the original Block Proof - * definition isn't used. Then we'll remove the original. - * - * ### Keywords - * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", - * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this - * document are to be interpreted as described in - * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in - * [RFC8174](https://www.ietf.org/rfc/rfc8174). - */ -syntax = "proto3"; - -package com.hedera.hapi.block.stream.output; - -// SPDX-License-Identifier: Apache-2.0 -option java_package = "com.hedera.hapi.block.stream.output.protoc"; -// <<>> This comment is special code for setting PBJ Compiler java package -option java_multiple_files = true; - -import "block/stream/record_file_item.proto"; - -/** - * TODO - */ -message BlockFooter { - - /** - * TODO - */ - bytes previous_block_root_hash = 1; - - /** - * TODO - */ - bytes root_hash_of_all_block_hashes_tree = 2; - - /** - * TODO - */ - bytes start_of_block_state_root_hash = 3; -} - -message TssSignedBlockProof { - bytes block_signature = 4; - - // TODO: probably will be deleted? - oneof verification_reference { - uint64 scheme_id = 6; - bytes verification_key = 7; // extracted from ledger ID? - } -} - -message SignedRecordFileProof { - /** - * A collection of RSA signatures from consensus nodes.
- * These signatures validate the hash of the record_file_contents field. - */ - repeated com.hedera.hapi.block.stream.RecordFileSignature record_file_signatures = 1; -} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java index 3aab313b31a0..5438eb5f3559 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java @@ -543,15 +543,7 @@ private T toView(@NonNull final BlockItemsTranslator translator, @NonNull fi } List logs = null; for (final var item : blockItems.subList(j, n)) { - if (item.hasTraceData()) { - final var traceData = item.traceDataOrThrow(); - if (traceData.hasEvmTraceData()) { - if (logs == null) { - logs = new ArrayList<>(); - } - logs.addAll(traceData.evmTraceDataOrThrow().logs()); - } - } + // TODO: new trace data implementation } return (T) switch (view) { @@ -677,7 +669,7 @@ public Output build(final boolean topLevel, @Nullable final List ba builder.logs(logs); } blockItems.add(BlockItem.newBuilder() - .traceData(TraceData.newBuilder().evmTraceData(builder)) + // TODO: re-add trace data .build()); } @@ -689,7 +681,7 @@ public Output build(final boolean topLevel, @Nullable final List ba .automaticTokenAssociations( automaticTokenAssociations.getLast().accountId()); blockItems.add(BlockItem.newBuilder() - .traceData(TraceData.newBuilder().autoAssociateTraceData(builder)) + // TODO: re-add trace data .build()); } // message submit trace data @@ -698,7 +690,7 @@ public Output build(final boolean topLevel, @Nullable final List ba .sequenceNumber(sequenceNumber) .runningHash(runningHash); blockItems.add(BlockItem.newBuilder() - .traceData(TraceData.newBuilder().submitMessageTraceData(builder)) + // TODO: re-add trace data .build()); } } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 938afa3e011c..de0263f1b525 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -91,6 +91,7 @@ import org.apache.logging.log4j.Logger; import org.hiero.base.concurrent.AbstractTask; import org.hiero.base.crypto.Hash; +import org.hiero.base.exceptions.NotImplementedException; import org.hiero.consensus.model.hashgraph.Round; @Singleton @@ -759,7 +760,10 @@ protected boolean onExecute() { STATE_CHANGES, ROUND_HEADER, BLOCK_HEADER, - TRACE_DATA -> { + BLOCK_FOOTER, + BLOCK_PROOF + // Also EndBlock? + -> { MessageDigest digest = sha384DigestOrThrow(); bytes.writeTo(digest); hash = ByteBuffer.wrap(digest.digest()); @@ -798,7 +802,7 @@ protected boolean onExecute() { } case TRANSACTION_OUTPUT, BLOCK_HEADER -> outputTreeHasher.addLeaf(hash); case STATE_CHANGES -> stateChangesHasher.addLeaf(hash); - case TRACE_DATA -> traceDataHasher.addLeaf(hash); + case BLOCK_FOOTER, BLOCK_PROOF -> throw new NotImplementedException(); } final BlockHeader header = item.blockHeader(); diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java index 0c70bca01cc9..a7e291cf2523 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java @@ -141,11 +141,12 @@ void testBlockItemsWithTraceAndOutput() { assertTrue(output.hasContractCall()); final var traceItem = blockItems.get(3); - assertTrue(traceItem.hasTraceData()); - final var trace = traceItem.traceDataOrThrow(); - assertTrue(trace.hasEvmTraceData()); - final var evmTrace = trace.evmTraceDataOrThrow(); - assertEquals(usages, evmTrace.contractSlotUsages()); + // TODO: assert trace data +// assertTrue(traceItem.hasTraceData()); +// final var trace = traceItem.traceDataOrThrow(); +// assertTrue(trace.hasEvmTraceData()); +// final var evmTrace = trace.evmTraceDataOrThrow(); +// assertEquals(usages, evmTrace.contractSlotUsages()); } @Test @@ -161,14 +162,15 @@ void testBlockItemsWithAdditionalAutomaticTokenAssociationTraceData() { final var blockItems = itemsBuilder.build(false, List.of()).blockItems(); final var traceItem = blockItems.get(2); - assertThat(traceItem.hasTraceData()).isTrue(); - final var trace = traceItem.traceDataOrThrow(); - - assertThat(trace.hasAutoAssociateTraceData()).isTrue(); - final var autoAssociateTraceData = trace.autoAssociateTraceData(); - assertThat(autoAssociateTraceData).isNotNull(); - assertThat(autoAssociateTraceData.automaticTokenAssociations().accountNum()) - .isEqualTo(2); + // TODO: assert trace data +// assertThat(traceItem.hasTraceData()).isTrue(); +// final var trace = traceItem.traceDataOrThrow(); +// +// assertThat(trace.hasAutoAssociateTraceData()).isTrue(); +// final var autoAssociateTraceData = trace.autoAssociateTraceData(); +// assertThat(autoAssociateTraceData).isNotNull(); +// assertThat(autoAssociateTraceData.automaticTokenAssociations().accountNum()) +// .isEqualTo(2); } @Test @@ -179,13 +181,14 @@ void testBlockItemsWithAdditionalSubmitMsgTraceData() { final var blockItems = itemsBuilder.build(false, List.of()).blockItems(); final var traceItem = blockItems.get(2); - assertThat(traceItem.hasTraceData()).isTrue(); - final var trace = traceItem.traceDataOrThrow(); - - assertThat(trace.hasSubmitMessageTraceData()).isTrue(); - final var submitMessageTraceData = trace.submitMessageTraceData(); - assertThat(submitMessageTraceData).isNotNull(); - assertThat(submitMessageTraceData.sequenceNumber()).isEqualTo(66); + // TODO: assert trace data +// assertThat(traceItem.hasTraceData()).isTrue(); +// final var trace = traceItem.traceDataOrThrow(); +// +// assertThat(trace.hasSubmitMessageTraceData()).isTrue(); +// final var submitMessageTraceData = trace.submitMessageTraceData(); +// assertThat(submitMessageTraceData).isNotNull(); +// assertThat(submitMessageTraceData.sequenceNumber()).isEqualTo(66); } @Test diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index d2ac32511bb4..39dd92e856f8 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -114,8 +114,8 @@ class BlockStreamManagerImplTest { private static final BlockItem FAKE_STATE_CHANGES = BlockItem.newBuilder() .stateChanges(StateChanges.newBuilder().consensusTimestamp(CONSENSUS_THEN)) .build(); - private static final BlockItem FAKE_RECORD_FILE_ITEM = - BlockItem.newBuilder().recordFile(RecordFileItem.DEFAULT).build(); + // TODO: remove, or replace with wrapped record file item + private static final BlockItem FAKE_RECORD_FILE_ITEM = null; private final InitialStateHash hashInfo = new InitialStateHash(completedFuture(ZERO_BLOCK_HASH), 0); @Mock From 0020f29900082c841a6381503a6697884de3690c Mon Sep 17 00:00:00 2001 From: Artem Derevets Date: Mon, 6 Oct 2025 17:04:59 +0200 Subject: [PATCH 20/63] feat: add block footer (#21356) Signed-off-by: artemderevets --- .../blocks/impl/BlockStreamManagerImpl.java | 30 ++- .../impl/BlockStreamManagerImplTest.java | 238 +++++++++++++++++- 2 files changed, 262 insertions(+), 6 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index de0263f1b525..c0ce13d67c2e 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -91,7 +91,6 @@ import org.apache.logging.log4j.Logger; import org.hiero.base.concurrent.AbstractTask; import org.hiero.base.crypto.Hash; -import org.hiero.base.exceptions.NotImplementedException; import org.hiero.consensus.model.hashgraph.Round; @Singleton @@ -473,6 +472,26 @@ public boolean endRound(@NonNull final State state, final long roundNum) { final var stateChangesHash = stateChangesHasher.rootHash().join(); + // TODO(#21210): Implement streaming merkle tree of all block hashes from genesis to N-1 + // For now, using NULL_HASH as placeholder until the historical block data infrastructure is ready. + final var blockHashesTreeRoot = NULL_HASH; + + // Create BlockFooter with the three essential hashes: + // 1. previousBlockRootHash - Root hash of the previous block (N-1) + // 2. rootHashOfAllBlockHashesTree - Streaming tree of all block hashes 0..N-1 (TODO: #21210) + // 3. startOfBlockStateRootHash - State hash at the beginning of current block + final var blockFooter = com.hedera.hapi.block.stream.output.BlockFooter.newBuilder() + .previousBlockRootHash(lastBlockHash) + .rootHashOfAllBlockHashesTree(blockHashesTreeRoot) + .startOfBlockStateRootHash(blockStartStateHash) + .build(); + + // Write BlockFooter to block stream (last item before BlockProof) + final var footerItem = + BlockItem.newBuilder().blockFooter(blockFooter).build(); + worker.addItem(footerItem); + worker.sync(); + // Compute depth two hashes final var depth2Node0 = combine(lastBlockHash, blockStartStateHash); final var depth2Node1 = combine(consensusHeaderHash, inputHash); @@ -759,9 +778,7 @@ protected boolean onExecute() { TRANSACTION_OUTPUT, STATE_CHANGES, ROUND_HEADER, - BLOCK_HEADER, - BLOCK_FOOTER, - BLOCK_PROOF + BLOCK_HEADER // Also EndBlock? -> { MessageDigest digest = sha384DigestOrThrow(); @@ -802,7 +819,10 @@ protected boolean onExecute() { } case TRANSACTION_OUTPUT, BLOCK_HEADER -> outputTreeHasher.addLeaf(hash); case STATE_CHANGES -> stateChangesHasher.addLeaf(hash); - case BLOCK_FOOTER, BLOCK_PROOF -> throw new NotImplementedException(); + case BLOCK_FOOTER, BLOCK_PROOF -> { + // BlockFooter and BlockProof are not included in any merkle tree + // They are metadata about the block, not part of the hashed content + } } final BlockHeader header = item.blockHeader(); diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index 39dd92e856f8..188e02a7c9cb 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -35,7 +35,6 @@ import static org.mockito.Mockito.withSettings; import com.hedera.hapi.block.stream.BlockItem; -import com.hedera.hapi.block.stream.RecordFileItem; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.block.stream.output.StateChanges; import com.hedera.hapi.block.stream.output.TransactionResult; @@ -72,11 +71,13 @@ import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -926,6 +927,241 @@ void eventHashMapIsClearedBetweenBlocks() { assertEquals(Optional.of(0), subject.getEventIndex(eventHash3)); } + @Test + @SuppressWarnings("unchecked") + void writesBlockFooterBeforeBlockProof() { + // Given a manager with a single round per block + givenSubjectWith( + 1, 0, blockStreamInfoWith(Bytes.EMPTY, CREATION_VERSION), platformStateWithFreezeTime(null), aWriter); + givenEndOfRoundSetup(); + + final AtomicReference footerItem = new AtomicReference<>(); + final AtomicReference proofItem = new AtomicReference<>(); + + doAnswer(invocationOnMock -> { + final var item = BlockItem.PROTOBUF.parse((Bytes) invocationOnMock.getArgument(1)); + if (item.hasBlockFooter()) { + footerItem.set(item); + } else if (item.hasBlockProof()) { + proofItem.set(item); + } + return aWriter; + }) + .when(aWriter) + .writePbjItemAndBytes(any(), any()); + + given(round.getRoundNum()).willReturn(ROUND_NO); + given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); + given(blockHashSigner.isReady()).willReturn(true); + given(blockHashSigner.schemeId()).willReturn(1L); + + // Set up the signature future to complete immediately + given(blockHashSigner.signFuture(any())).willReturn(mockSigningFuture); + doAnswer(invocationOnMock -> { + final Consumer consumer = invocationOnMock.getArgument(0); + consumer.accept(FIRST_FAKE_SIGNATURE); + return null; + }) + .when(mockSigningFuture) + .thenAcceptAsync(any()); + + // Initialize hash and start a round + subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); + subject.startRound(round, state); + + // Write some items + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.writeItem(FAKE_TRANSACTION_RESULT); + subject.writeItem(FAKE_STATE_CHANGES); + + // End the round + subject.endRound(state, ROUND_NO); + + // Verify BlockFooter was written + assertNotNull(footerItem.get(), "BlockFooter should be written"); + assertTrue(footerItem.get().hasBlockFooter()); + + final var footer = footerItem.get().blockFooterOrThrow(); + assertNotNull(footer.previousBlockRootHash(), "Previous block root hash should be set"); + // TODO(#21210): Currently using NULL_HASH placeholder for block hashes tree + // Will be replaced when streaming merkle tree of all block hashes is implemented + assertEquals( + BlockStreamManagerImpl.NULL_HASH, + footer.rootHashOfAllBlockHashesTree(), + "Block hashes tree root should be NULL_HASH until #21210 is implemented"); + assertNotNull(footer.startOfBlockStateRootHash(), "Start of block state root hash should be set"); + + // Verify BlockProof was also written + assertNotNull(proofItem.get(), "BlockProof should be written"); + assertTrue(proofItem.get().hasBlockProof()); + } + + @Test + @SuppressWarnings("unchecked") + void blockFooterContainsCorrectHashValues() { + // Given a manager with a single round per block + givenSubjectWith( + 1, 0, blockStreamInfoWith(Bytes.EMPTY, CREATION_VERSION), platformStateWithFreezeTime(null), aWriter); + givenEndOfRoundSetup(); + + final AtomicReference footerItem = new AtomicReference<>(); + + doAnswer(invocationOnMock -> { + final var item = BlockItem.PROTOBUF.parse((Bytes) invocationOnMock.getArgument(1)); + if (item.hasBlockFooter()) { + footerItem.set(item); + } + return aWriter; + }) + .when(aWriter) + .writePbjItemAndBytes(any(), any()); + + given(round.getRoundNum()).willReturn(ROUND_NO); + given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); + given(blockHashSigner.isReady()).willReturn(true); + given(blockHashSigner.schemeId()).willReturn(1L); + + // Set up the signature future + given(blockHashSigner.signFuture(any())).willReturn(mockSigningFuture); + doAnswer(invocationOnMock -> { + final Consumer consumer = invocationOnMock.getArgument(0); + consumer.accept(FIRST_FAKE_SIGNATURE); + return null; + }) + .when(mockSigningFuture) + .thenAcceptAsync(any()); + + // Initialize with known hash and start round + subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); + subject.startRound(round, state); + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.endRound(state, ROUND_NO); + + // Verify BlockFooter hash values + assertNotNull(footerItem.get(), "BlockFooter should be written"); + final var footer = footerItem.get().blockFooterOrThrow(); + + // Verify previousBlockRootHash matches the last block hash + assertEquals( + N_MINUS_2_BLOCK_HASH, + footer.previousBlockRootHash(), + "Previous block root hash should match initialized last block hash"); + + // Verify rootHashOfAllBlockHashesTree is NULL_HASH (placeholder) + assertEquals( + BlockStreamManagerImpl.NULL_HASH, + footer.rootHashOfAllBlockHashesTree(), + "Block hashes tree root should be NULL_HASH placeholder"); + + // Verify startOfBlockStateRootHash is set + assertEquals( + FAKE_START_OF_BLOCK_STATE_HASH.getBytes(), + footer.startOfBlockStateRootHash(), + "Start of block state root hash should match expected value"); + } + + @Test + @SuppressWarnings("unchecked") + void blockFooterWrittenForEachBlock() { + // Given a manager with a single round per block + givenSubjectWith( + 1, + 0, + blockStreamInfoWith(Bytes.EMPTY, CREATION_VERSION), + platformStateWithFreezeTime(null), + aWriter, + bWriter); + givenEndOfRoundSetup(); + + final List footerItems = new ArrayList<>(); + + doAnswer(invocationOnMock -> { + final var item = BlockItem.PROTOBUF.parse((Bytes) invocationOnMock.getArgument(1)); + if (item.hasBlockFooter()) { + footerItems.add(item); + } + return aWriter; + }) + .when(aWriter) + .writePbjItemAndBytes(any(), any()); + + doAnswer(invocationOnMock -> { + final var item = BlockItem.PROTOBUF.parse((Bytes) invocationOnMock.getArgument(1)); + if (item.hasBlockFooter()) { + footerItems.add(item); + } + return bWriter; + }) + .when(bWriter) + .writePbjItemAndBytes(any(), any()); + + given(round.getRoundNum()).willReturn(ROUND_NO); + given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); + given(blockHashSigner.isReady()).willReturn(true); + given(blockHashSigner.schemeId()).willReturn(1L); + + // Set up the signature futures + final CompletableFuture firstSignature = (CompletableFuture) mock(CompletableFuture.class); + final CompletableFuture secondSignature = (CompletableFuture) mock(CompletableFuture.class); + given(blockHashSigner.signFuture(any())).willReturn(firstSignature).willReturn(secondSignature); + + // Initialize and create first block + subject.initLastBlockHash(FAKE_RESTART_BLOCK_HASH); + subject.startRound(round, state); + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.endRound(state, ROUND_NO); + + // Create second block + given(round.getRoundNum()).willReturn(ROUND_NO + 1); + given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW.plusSeconds(1)); + given(notification.round()).willReturn(ROUND_NO); + given(notification.hash()).willReturn(FAKE_START_OF_BLOCK_STATE_HASH); + subject.notify(notification); + subject.startRound(round, state); + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.endRound(state, ROUND_NO + 1); + + // Verify BlockFooter was written for each block + assertEquals(2, footerItems.size(), "Should have written BlockFooter for each block"); + + // Verify both are valid BlockFooters + assertTrue(footerItems.get(0).hasBlockFooter(), "First item should be BlockFooter"); + assertTrue(footerItems.get(1).hasBlockFooter(), "Second item should be BlockFooter"); + } + + @Test + void blockFooterNotWrittenWhenBlockNotClosed() { + // Given a manager with 2 rounds per block + givenSubjectWith( + 2, 0, blockStreamInfoWith(Bytes.EMPTY, CREATION_VERSION), platformStateWithFreezeTime(null), aWriter); + givenEndOfRoundSetup(); + + final AtomicBoolean footerWritten = new AtomicBoolean(false); + + doAnswer(invocationOnMock -> { + final var item = BlockItem.PROTOBUF.parse((Bytes) invocationOnMock.getArgument(1)); + if (item.hasBlockFooter()) { + footerWritten.set(true); + } + return aWriter; + }) + .when(aWriter) + .writePbjItemAndBytes(any(), any()); + + given(round.getRoundNum()).willReturn(ROUND_NO); + given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); + given(blockHashSigner.isReady()).willReturn(true); + + // Initialize and start first round (block not yet closed) + subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); + subject.startRound(round, state); + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.endRound(state, ROUND_NO); + + // Verify BlockFooter was NOT written (block needs 2 rounds) + assertFalse(footerWritten.get(), "BlockFooter should not be written until block is closed"); + } + private void givenSubjectWith( final int roundsPerBlock, final int blockPeriod, From 8f5795122bcc243584785e8207d3216fab7e0899 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 6 Oct 2025 21:59:53 -0600 Subject: [PATCH 21/63] Fix compilation errors Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 85 +++++++++++++++++-- 1 file changed, 79 insertions(+), 6 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index f7ca563c7a4b..314a967b44b3 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -207,19 +207,70 @@ message BlockItem { com.hedera.hapi.block.stream.output.StateChanges state_changes = 7; /** - * TODO + * Verification data for an item filtered from the stream.
+ * This is a hash for a merkle tree node where the contents of that + * part of the merkle tree have been removed from this stream. + *

+ * Items of this type SHALL NOT be present in the full (unfiltered) + * block stream.
+ * Items of this type SHALL replace any item removed from a partial + * (filtered) block stream.
+ * Presence of `filtered_item` entries SHALL NOT prevent verification + * of a block, but MAY preclude verification or reconstruction of + * consensus state.
+ */ + FilteredItemHash filtered_item_hash = 8; + + /** + * A signed block proof.
+ * The signed merkle proof for this block. This will validate + * a "virtual" merkle tree containing the previous block "virtual" + * root, an "input" subtree, an "output" subtree, and + * a "state changes" subtree. + *

+ * This item is not part of the block stream hash chain/tree, and + * MUST follow after the end of a block. */ - FilteredSingleItem filtered_single_item = 8; + BlockProof block_proof = 9; /** - * TODO + * A record file and associated data. + *

+ * This MUST contain a single Record file, associated Sidecar files, + * and data from related Signature files. + * If this item is present, special treatment is + * REQUIRED for this block. + *

    + *
  • The block SHALL NOT have a `BlockHeader`.
    • + *
  • The block SHALL NOT have a `BlockProof`.
    • + *
  • The block SHALL contain _exactly one_ `RecordFileItem`.
    • + *
  • The block SHALL NOT contain any item other than a + * `RecordFileItem`.
    • + *
  • The content of the `RecordFileItem` MUST be validated using + * the signature data and content provided within according to + * the process used for Record Files prior to the creation + * of Block Stream.
    • + *
*/ - com.hedera.hapi.block.stream.output.BlockFooter block_footer = 9; + RecordFileItem record_file = 10; /** - * TODO + * A trace data. + *

+ * Any informational trace data MAY be described by + * stream items of this type.
*/ - BlockProof block_proof = 10; + com.hedera.hapi.block.stream.trace.TraceData trace_data = 11; + + /** + * TODO + */ + FilteredSingleItem filtered_single_item = 20; + + /** + * TODO + */ + com.hedera.hapi.block.stream.output.BlockFooter block_footer = 21; } } @@ -252,6 +303,28 @@ enum SubMerkleTree { * of a block, but MAY preclude verification or reconstruction * of consensus state.
*/ +message FilteredItemHash { + /** + * A hash of an item filtered from the stream. + *

+ * The hash algorithm used MUST match the hash algorithm specified in + * the block header for the containing block.
+ * This field is REQUIRED. + */ + bytes item_hash = 1; + + /** + * A record of the merkle path to the item that was filtered + * from the stream.
+ * This path begins at the root of the block proof merkle tree. + *

+ * This REQUIRED field SHALL describe the full path in the virtual + * merkle tree constructed for the block proof that contained the + * item filtered from the stream. + */ + uint64 filtered_path = 3; +} + message FilteredSingleItem { /** * A hash of an item filtered from the stream. From 981aeb875e3111c91aa80f7629b2346b423954ac Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 6 Oct 2025 22:02:24 -0600 Subject: [PATCH 22/63] spotless Signed-off-by: Matt Hess --- .../app/blocks/impl/BlockStreamBuilder.java | 8 +- .../blocks/impl/BlockStreamManagerImpl.java | 4 +- .../impl/IncrementalStreamingHasher.java | 190 +++++++++--------- .../app/blocks/BlockStreamBuilderTest.java | 46 ++--- .../impl/BlockStreamManagerImplTest.java | 2 +- 5 files changed, 125 insertions(+), 125 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java index 5438eb5f3559..435a47128883 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java @@ -543,7 +543,7 @@ private T toView(@NonNull final BlockItemsTranslator translator, @NonNull fi } List logs = null; for (final var item : blockItems.subList(j, n)) { - // TODO: new trace data implementation + // TODO: new trace data implementation } return (T) switch (view) { @@ -669,7 +669,7 @@ public Output build(final boolean topLevel, @Nullable final List ba builder.logs(logs); } blockItems.add(BlockItem.newBuilder() - // TODO: re-add trace data + // TODO: re-add trace data .build()); } @@ -681,7 +681,7 @@ public Output build(final boolean topLevel, @Nullable final List ba .automaticTokenAssociations( automaticTokenAssociations.getLast().accountId()); blockItems.add(BlockItem.newBuilder() - // TODO: re-add trace data + // TODO: re-add trace data .build()); } // message submit trace data @@ -690,7 +690,7 @@ public Output build(final boolean topLevel, @Nullable final List ba .sequenceNumber(sequenceNumber) .runningHash(runningHash); blockItems.add(BlockItem.newBuilder() - // TODO: re-add trace data + // TODO: re-add trace data .build()); } } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index c0ce13d67c2e..191185e1d8e2 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -779,8 +779,8 @@ protected boolean onExecute() { STATE_CHANGES, ROUND_HEADER, BLOCK_HEADER - // Also EndBlock? - -> { + // Also EndBlock? + -> { MessageDigest digest = sha384DigestOrThrow(); bytes.writeTo(digest); hash = ByteBuffer.wrap(digest.digest()); diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java index 10addc43cb03..e5231952de68 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java @@ -14,107 +14,107 @@ *

This is not thread safe, it is assumed use by single thread.

*/ public class IncrementalStreamingHasher { - /** Prefix byte for hash contents for leaf nodes. */ - private static final byte[] LEAF_PREFIX = new byte[] {0}; - /** Prefix byte for hash contents for internal nodes. */ - private static final byte[] INTERNAL_NODE_PREFIX = new byte[] {2}; - /** The hashing algorithm used for computing the hashes. */ - private final MessageDigest digest; - /** A list to store intermediate hashes as we build the tree. */ - private final LinkedList hashList = new LinkedList<>(); - /** The count of leaves in the tree. */ - private long leafCount = 0; + /** Prefix byte for hash contents for leaf nodes. */ + private static final byte[] LEAF_PREFIX = new byte[] {0}; + /** Prefix byte for hash contents for internal nodes. */ + private static final byte[] INTERNAL_NODE_PREFIX = new byte[] {2}; + /** The hashing algorithm used for computing the hashes. */ + private final MessageDigest digest; + /** A list to store intermediate hashes as we build the tree. */ + private final LinkedList hashList = new LinkedList<>(); + /** The count of leaves in the tree. */ + private long leafCount = 0; - /** Create a new StreamingHasher with an empty state. */ - public IncrementalStreamingHasher() { - try { - digest = MessageDigest.getInstance("SHA-384"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } + /** Create a new StreamingHasher with an empty state. */ + public IncrementalStreamingHasher() { + try { + digest = MessageDigest.getInstance("SHA-384"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } - /** - * Create a StreamingHasher with an existing intermediate hashing state. - * This allows resuming hashing from a previous state. - * - * @param intermediateHashingState the intermediate hashing state - */ - public IncrementalStreamingHasher(List intermediateHashingState) { - this(); - this.hashList.addAll(intermediateHashingState); - } + /** + * Create a StreamingHasher with an existing intermediate hashing state. + * This allows resuming hashing from a previous state. + * + * @param intermediateHashingState the intermediate hashing state + */ + public IncrementalStreamingHasher(List intermediateHashingState) { + this(); + this.hashList.addAll(intermediateHashingState); + } - /** - * Add a new leaf to the Merkle tree. - * - * @param data the data for the new leaf - */ - public void addLeaf(byte[] data) { - final long i = leafCount; - final byte[] e = hashLeaf(data); - hashList.add(e); - for (long n=i; (n & 1L) == 1; n >>= 1) { - final byte[] y = hashList.removeLast(); - final byte[] x = hashList.removeLast(); - hashList.add(hashInternalNode(x, y)); - } - leafCount ++; - } + /** + * Add a new leaf to the Merkle tree. + * + * @param data the data for the new leaf + */ + public void addLeaf(byte[] data) { + final long i = leafCount; + final byte[] e = hashLeaf(data); + hashList.add(e); + for (long n = i; (n & 1L) == 1; n >>= 1) { + final byte[] y = hashList.removeLast(); + final byte[] x = hashList.removeLast(); + hashList.add(hashInternalNode(x, y)); + } + leafCount++; + } - /** - * Compute the Merkle tree root hash from the current state. This does not modify the internal state, so can be - * called at any time and more leaves can be added afterward. - * - * @return the Merkle tree root hash - */ - public byte[] computeRootHash() { - byte[] merkleRootHash = hashList.getLast(); - for (int i = hashList.size() - 2; i >= 0; i--) { - merkleRootHash = hashInternalNode(hashList.get(i), merkleRootHash); - } - return merkleRootHash; - } + /** + * Compute the Merkle tree root hash from the current state. This does not modify the internal state, so can be + * called at any time and more leaves can be added afterward. + * + * @return the Merkle tree root hash + */ + public byte[] computeRootHash() { + byte[] merkleRootHash = hashList.getLast(); + for (int i = hashList.size() - 2; i >= 0; i--) { + merkleRootHash = hashInternalNode(hashList.get(i), merkleRootHash); + } + return merkleRootHash; + } - /** - * Get the current intermediate hashing state. This can be used to save the state and resume hashing later. - * - * @return the intermediate hashing state - */ - public List intermediateHashingState() { - return hashList; - } + /** + * Get the current intermediate hashing state. This can be used to save the state and resume hashing later. + * + * @return the intermediate hashing state + */ + public List intermediateHashingState() { + return hashList; + } - /** - * Get the number of leaves added to the tree so far. - * - * @return the number of leaves - */ - public long leafCount() { - return leafCount; - } + /** + * Get the number of leaves added to the tree so far. + * + * @return the number of leaves + */ + public long leafCount() { + return leafCount; + } - /** - * Hash a leaf node with the appropriate prefix. - * - * @param leafData the data of the leaf - * @return the hash of the leaf node - */ - private byte[] hashLeaf(final byte[] leafData) { - digest.update(LEAF_PREFIX); - return digest.digest(leafData); - } + /** + * Hash a leaf node with the appropriate prefix. + * + * @param leafData the data of the leaf + * @return the hash of the leaf node + */ + private byte[] hashLeaf(final byte[] leafData) { + digest.update(LEAF_PREFIX); + return digest.digest(leafData); + } - /** - * Hash an internal node by combining the hashes of its two children with the appropriate prefix. - * - * @param firstChild the hash of the first child - * @param secondChild the hash of the second child - * @return the hash of the internal node - */ - private byte[] hashInternalNode(final byte[] firstChild, final byte[] secondChild) { - digest.update(INTERNAL_NODE_PREFIX); - digest.update(firstChild); - return digest.digest(secondChild); - } + /** + * Hash an internal node by combining the hashes of its two children with the appropriate prefix. + * + * @param firstChild the hash of the first child + * @param secondChild the hash of the second child + * @return the hash of the internal node + */ + private byte[] hashInternalNode(final byte[] firstChild, final byte[] secondChild) { + digest.update(INTERNAL_NODE_PREFIX); + digest.update(firstChild); + return digest.digest(secondChild); + } } diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java index a7e291cf2523..5485074c0f9d 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java @@ -141,12 +141,12 @@ void testBlockItemsWithTraceAndOutput() { assertTrue(output.hasContractCall()); final var traceItem = blockItems.get(3); - // TODO: assert trace data -// assertTrue(traceItem.hasTraceData()); -// final var trace = traceItem.traceDataOrThrow(); -// assertTrue(trace.hasEvmTraceData()); -// final var evmTrace = trace.evmTraceDataOrThrow(); -// assertEquals(usages, evmTrace.contractSlotUsages()); + // TODO: assert trace data + // assertTrue(traceItem.hasTraceData()); + // final var trace = traceItem.traceDataOrThrow(); + // assertTrue(trace.hasEvmTraceData()); + // final var evmTrace = trace.evmTraceDataOrThrow(); + // assertEquals(usages, evmTrace.contractSlotUsages()); } @Test @@ -162,15 +162,15 @@ void testBlockItemsWithAdditionalAutomaticTokenAssociationTraceData() { final var blockItems = itemsBuilder.build(false, List.of()).blockItems(); final var traceItem = blockItems.get(2); - // TODO: assert trace data -// assertThat(traceItem.hasTraceData()).isTrue(); -// final var trace = traceItem.traceDataOrThrow(); -// -// assertThat(trace.hasAutoAssociateTraceData()).isTrue(); -// final var autoAssociateTraceData = trace.autoAssociateTraceData(); -// assertThat(autoAssociateTraceData).isNotNull(); -// assertThat(autoAssociateTraceData.automaticTokenAssociations().accountNum()) -// .isEqualTo(2); + // TODO: assert trace data + // assertThat(traceItem.hasTraceData()).isTrue(); + // final var trace = traceItem.traceDataOrThrow(); + // + // assertThat(trace.hasAutoAssociateTraceData()).isTrue(); + // final var autoAssociateTraceData = trace.autoAssociateTraceData(); + // assertThat(autoAssociateTraceData).isNotNull(); + // assertThat(autoAssociateTraceData.automaticTokenAssociations().accountNum()) + // .isEqualTo(2); } @Test @@ -181,14 +181,14 @@ void testBlockItemsWithAdditionalSubmitMsgTraceData() { final var blockItems = itemsBuilder.build(false, List.of()).blockItems(); final var traceItem = blockItems.get(2); - // TODO: assert trace data -// assertThat(traceItem.hasTraceData()).isTrue(); -// final var trace = traceItem.traceDataOrThrow(); -// -// assertThat(trace.hasSubmitMessageTraceData()).isTrue(); -// final var submitMessageTraceData = trace.submitMessageTraceData(); -// assertThat(submitMessageTraceData).isNotNull(); -// assertThat(submitMessageTraceData.sequenceNumber()).isEqualTo(66); + // TODO: assert trace data + // assertThat(traceItem.hasTraceData()).isTrue(); + // final var trace = traceItem.traceDataOrThrow(); + // + // assertThat(trace.hasSubmitMessageTraceData()).isTrue(); + // final var submitMessageTraceData = trace.submitMessageTraceData(); + // assertThat(submitMessageTraceData).isNotNull(); + // assertThat(submitMessageTraceData.sequenceNumber()).isEqualTo(66); } @Test diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index 188e02a7c9cb..5bba278d2bf4 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -115,7 +115,7 @@ class BlockStreamManagerImplTest { private static final BlockItem FAKE_STATE_CHANGES = BlockItem.newBuilder() .stateChanges(StateChanges.newBuilder().consensusTimestamp(CONSENSUS_THEN)) .build(); - // TODO: remove, or replace with wrapped record file item + // TODO: remove, or replace with wrapped record file item private static final BlockItem FAKE_RECORD_FILE_ITEM = null; private final InitialStateHash hashInfo = new InitialStateHash(completedFuture(ZERO_BLOCK_HASH), 0); From 10b8d0bfd74a6df65fa3252978aafd5e093c205d Mon Sep 17 00:00:00 2001 From: Zhivko Kelchev Date: Thu, 16 Oct 2025 18:14:19 +0300 Subject: [PATCH 23/63] feat: Send end block request (#21413) Signed-off-by: Zhivko Kelchev --- .../app/blocks/impl/streaming/BlockState.java | 14 +++ .../app/blocks/BlockStreamBuilderTest.java | 1 - .../simulator/SimulatedBlockNodeServer.java | 96 +++++++++++-------- .../bdd/suites/blocknode/BlockNodeSuite.java | 31 ++++++ hiero-dependency-versions/build.gradle.kts | 2 +- 5 files changed, 101 insertions(+), 43 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/streaming/BlockState.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/streaming/BlockState.java index 9b83b820f336..7aaeb8b8eeb0 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/streaming/BlockState.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/streaming/BlockState.java @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.hiero.block.api.BlockEnd; import org.hiero.block.api.BlockItemSet; import org.hiero.block.api.PublishStreamRequest; @@ -292,6 +293,7 @@ public synchronized void processPendingItems(final int batchSize) { final Iterator it = pendingItems.iterator(); boolean forceCreation = false; + boolean sendEndOfBlock = false; while (it.hasNext()) { final BlockItem item = it.next(); blockItems.add(item); @@ -320,6 +322,8 @@ && isPreProofItemReceived(item.stateChangesOrElse(StateChanges.DEFAULT))) { } else if (item.hasBlockProof()) { if (proofItemInfo.packedInRequest(index)) { forceCreation = true; + // send end of block request if the proof is packed + sendEndOfBlock = true; logger.trace("[Block {}] Block proof packed in request #{}", blockNumber, index); } else { logger.warn( @@ -344,6 +348,16 @@ && isPreProofItemReceived(item.stateChangesOrElse(StateChanges.DEFAULT))) { logger.trace("[Block {}] Created new request (index={}, numItems={})", blockNumber, index, blockItems.size()); + if (sendEndOfBlock) { + final var eobRequest = PublishStreamRequest.newBuilder() + .endOfBlock(BlockEnd.newBuilder().blockNumber(blockNumber)) + .build(); + final var eobRequestIndex = requestIdxCtr.getAndIncrement(); + final RequestWrapper rsEnd = new RequestWrapper(eobRequestIndex, eobRequest, new AtomicBoolean(false)); + requestsByIndex.put(eobRequestIndex, rsEnd); + logger.trace("[Block {}] Created new request (index={}, BlockEnd)", blockNumber, eobRequestIndex); + } + if (!pendingItems.isEmpty()) { processPendingItems(batchSize); } diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java index 5485074c0f9d..d6b6e120440e 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java @@ -10,7 +10,6 @@ import static com.hedera.node.app.spi.workflows.HandleContext.TransactionCategory.USER; import static com.hedera.node.app.spi.workflows.record.StreamBuilder.ReversingBehavior.REVERSIBLE; import static com.hedera.node.app.spi.workflows.record.StreamBuilder.SignedTxCustomizer.NOOP_SIGNED_TX_CUSTOMIZER; -import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/hedera/simulator/SimulatedBlockNodeServer.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/hedera/simulator/SimulatedBlockNodeServer.java index 1c8aaa971e9e..41338213b339 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/hedera/simulator/SimulatedBlockNodeServer.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/hedera/simulator/SimulatedBlockNodeServer.java @@ -83,8 +83,8 @@ public class SimulatedBlockNodeServer { // Locks for synchronizing access to block tracking data structures private final ReadWriteLock blockTrackingLock = new ReentrantReadWriteLock(); - // Track all block numbers for which we have received proofs - private final Set blocksWithProofs = ConcurrentHashMap.newKeySet(); + // Track all block numbers for which we have received end block + private final Set blocksWithEndBlock = ConcurrentHashMap.newKeySet(); // Track all block numbers for which we have received headers but not yet proofs private final Set blocksWithHeadersOnly = ConcurrentHashMap.newKeySet(); @@ -250,8 +250,8 @@ public long getLastVerifiedBlockNumber() { public boolean hasReceivedBlock(final long blockNumber) { blockTrackingLock.readLock().lock(); try { - // A block is considered received only if we have its proof - return blocksWithProofs.contains(blockNumber); + // A block is considered received only if we have its end block + return blocksWithEndBlock.contains(blockNumber); } finally { blockTrackingLock.readLock().unlock(); } @@ -267,8 +267,8 @@ public boolean hasReceivedBlock(final long blockNumber) { public Set getReceivedBlockNumbers() { blockTrackingLock.readLock().lock(); try { - // Return only blocks for which we have proofs - return Set.copyOf(blocksWithProofs); + // Return only blocks for which we have the end block + return Set.copyOf(blocksWithEndBlock); } finally { blockTrackingLock.readLock().unlock(); } @@ -382,8 +382,8 @@ public void onNext(final PublishStreamRequest request) { port, replies.hashCode()); - // Requirement 3: Check if block already exists (header AND proof received) - if (blocksWithProofs.contains(blockNumber)) { + // Requirement 3: Check if block already exists (header AND end block received) + if (blocksWithEndBlock.contains(blockNumber)) { log.warn( "Block {} already fully received (header+proof). Sending BlockAcknowledgement to stream {} on port {}.", blockNumber, @@ -461,42 +461,56 @@ public void onNext(final PublishStreamRequest request) { // Continue to the next BlockItem in the request continue; } - - // Mark block as fully received - blocksWithHeadersOnly.remove(blockNumber); - blocksWithProofs.add(blockNumber); - streamingBlocks.remove(blockNumber); // No longer streaming this specific block - - // Update last verified block number atomically - final long newLastVerified = lastVerifiedBlockNumber.updateAndGet( - currentMax -> Math.max(currentMax, blockNumber)); - log.info( - "Block {} fully received (header+proof) on port {} from stream {}. Last verified block updated to: {}", - blockNumber, - port, - replies.hashCode(), - newLastVerified); - - // Requirement 2: Send BlockAcknowledgement to ALL connected pipelines - log.info( - "Broadcasting BlockAcknowledgement for block {} to {} active streams on port {}", - blockNumber, - activeStreams.size(), - port); - for (final Pipeline pipeline : activeStreams) { - if (highLatency) { - // If the simulator is set to be with high latency, delay acknowledgements - // with 1500 ms (assuming CN considers 1000 ms delays as high latency) - Thread.sleep(1500); - } - - buildAndSendBlockAcknowledgement(blockNumber, pipeline); + } + } // End of loop through BlockItems + } else if (request.hasEndOfBlock()) { + final var blockNumber = request.endOfBlockOrThrow().blockNumber(); + if (currentBlockNumber == null + || currentBlockNumber != blockNumber + || !streamingBlocks.containsKey(blockNumber) + || streamingBlocks.get(blockNumber) != replies) { + log.info( + "Received EndBlock for block {} from stream {} on port {}", + blockNumber, + replies.hashCode(), + port); + + // Mark block as fully received + blocksWithHeadersOnly.remove(blockNumber); + blocksWithEndBlock.add(blockNumber); + streamingBlocks.remove(blockNumber); // No longer streaming this specific block + + // Update last verified block number atomically + final long newLastVerified = lastVerifiedBlockNumber.updateAndGet( + currentMax -> Math.max(currentMax, blockNumber)); + log.info( + "Block {} fully received (header+proof) on port {} from stream {}. Last verified block updated to: {}", + blockNumber, + port, + replies.hashCode(), + newLastVerified); + + // Requirement 2: Send BlockAcknowledgement to ALL connected pipelines + log.info( + "Broadcasting BlockAcknowledgement for block {} to {} active streams on port {}", + blockNumber, + activeStreams.size(), + port); + + // send acknowledgment + for (final Pipeline pipeline : activeStreams) { + if (highLatency) { + // If the simulator is set to be with high latency, delay acknowledgements + // with 1500 ms (assuming CN considers 1000 ms delays as high latency) + Thread.sleep(1500); } - // Reset currentBlockNumber for this stream, as it finished sending this block - currentBlockNumber = null; + buildAndSendBlockAcknowledgement(blockNumber, pipeline); } - } // End of loop through BlockItems + + // Reset currentBlockNumber for this stream, as it finished sending this block + currentBlockNumber = null; + } } } catch (InterruptedException e) { log.warn("Interrupted while waiting for BlockAcknowledgement", e); diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java index 421e222d0c4c..9f6a6058c19c 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java @@ -1015,4 +1015,35 @@ private Stream validateHappyPath(int blocksToWait) { assertBlockNodeCommsLogDoesNotContain( byNodeId(0), "Block node has exceeded high latency threshold", Duration.ofSeconds(0))); } + + @HapiTest + @HapiBlockNode( + networkSize = 1, + blockNodeConfigs = {@BlockNodeConfig(nodeId = 0, mode = BlockNodeMode.SIMULATOR)}, + subProcessNodeConfigs = { + @SubProcessNodeConfig( + nodeId = 0, + blockNodeIds = {0}, + blockNodePriorities = {0}, + applicationPropertiesOverrides = { + "blockStream.streamMode", "BOTH", + "blockStream.writerMode", "FILE_AND_GRPC" + }) + }) + @Order(13) + final Stream node0SendEndOfBlockHappyPath() { + final AtomicReference timeRef = new AtomicReference<>(); + return hapiTest( + doingContextual(spec -> timeRef.set(Instant.now())), + waitUntilNextBlocks(10).withBackgroundTraffic(true), + // assert no errors + assertHgcaaLogDoesNotContain(byNodeId(0), "ERROR", Duration.ofSeconds(5)), + sourcingContextual(spec -> assertHgcaaLogContainsTimeframe( + byNodeId(0), + timeRef::get, + Duration.ofMinutes(1), + Duration.ofMinutes(1), + // Should send END_OF_BLOCK requests + "Sending request to block node (type=END_OF_BLOCK)"))); + } } diff --git a/hiero-dependency-versions/build.gradle.kts b/hiero-dependency-versions/build.gradle.kts index ec1b5d1e6b92..78196822a907 100644 --- a/hiero-dependency-versions/build.gradle.kts +++ b/hiero-dependency-versions/build.gradle.kts @@ -29,7 +29,7 @@ val log4j = "2.25.0" val mockito = "5.18.0" val pbj = "0.12.1" // ATTENTION: keep in sync with plugin version in 'hapi/hapi/build.gradle.kts' val protobuf = "4.31.1" -val blockNodeProtobufSources = "0.17.1" +val blockNodeProtobufSources = "0.20.0-rc1" val testContainers = "1.21.3" val tuweni = "2.4.2" val webcompare = "2.1.8" From f6fea5845c13340d8ce13ef639512dcf917fc533 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Thu, 23 Oct 2025 01:54:41 -0600 Subject: [PATCH 24/63] Rework Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 238 +++++++-------- .../main/proto/block/stream/block_proof.proto | 6 +- .../block/stream/output/block_footer.proto | 7 +- .../main/proto/block/stream/state_proof.proto | 5 +- .../state/blockstream/block_stream_info.proto | 272 +++++++++-------- .../blocks/impl/BlockStreamManagerImpl.java | 288 ++++++++++-------- .../impl/BlockStreamManagerImplTest.java | 2 - .../bdd/suites/blocknode/BlockNodeSuite.java | 2 + 8 files changed, 429 insertions(+), 391 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index 54784bbecb62..acddc7ff03a7 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -140,138 +140,138 @@ import "block/stream/output/block_footer.proto"; * - `record_file` */ message BlockItem { - // Reserved for future items that require separate handling for block hash purposes. - reserved 12,13,14,15,16,17,18,19; + // Reserved for future items that require separate handling for block hash purposes. + reserved 12,13,14,15,16,17,18,19; - oneof item { - /** - * An header for the block, marking the start of a new block. - */ - com.hedera.hapi.block.stream.output.BlockHeader block_header = 1; + oneof item { + /** + * An header for the block, marking the start of a new block. + */ + com.hedera.hapi.block.stream.output.BlockHeader block_header = 1; - /** - * An header emitted at the start of a new network "event". - *

- * This item SHALL contain the properties relevant to a single - * gossip event. - */ - com.hedera.hapi.block.stream.input.EventHeader event_header = 2; + /** + * An header emitted at the start of a new network "event". + *

+ * This item SHALL contain the properties relevant to a single + * gossip event. + */ + com.hedera.hapi.block.stream.input.EventHeader event_header = 2; - /** - * An header emitted at the start of a new consensus "round". - *

- * This item SHALL contain the properties relevant to a single - * consensus round. - */ - com.hedera.hapi.block.stream.input.RoundHeader round_header = 3; + /** + * An header emitted at the start of a new consensus "round". + *

+ * This item SHALL contain the properties relevant to a single + * consensus round. + */ + com.hedera.hapi.block.stream.input.RoundHeader round_header = 3; - /** - * A single transaction. - *

- * This item SHALL contain the serialized bytes of a - * single proto.SignedTransaction.
- * This item MUST NOT contain data for more than one - * `SignedTransaction`. - */ - bytes signed_transaction = 4; + /** + * A single transaction. + *

+ * This item SHALL contain the serialized bytes of a + * single proto.SignedTransaction.
+ * This item MUST NOT contain data for more than one + * `SignedTransaction`. + */ + bytes signed_transaction = 4; - /** - * The result of running a transaction. - *

- * This item SHALL be present immediately after an - * `signed_transaction` item.
- * This item MAY be redacted in some circumstances, and SHALL be - * replaced with a `filtered_item` if removed. - */ - com.hedera.hapi.block.stream.output.TransactionResult transaction_result = 5; + /** + * The result of running a transaction. + *

+ * This item SHALL be present immediately after an + * `signed_transaction` item.
+ * This item MAY be redacted in some circumstances, and SHALL be + * replaced with a `filtered_item` if removed. + */ + com.hedera.hapi.block.stream.output.TransactionResult transaction_result = 5; - /** - * A transaction output. - *

- * This item MAY not be present if a transaction does not produce - * an output.
- * If a transaction does produce an output that is not reflected - * in state changes, then this item MUST be present after the - * `transaction_result` for that transaction. - */ - com.hedera.hapi.block.stream.output.TransactionOutput transaction_output = 6; + /** + * A transaction output. + *

+ * This item MAY not be present if a transaction does not produce + * an output.
+ * If a transaction does produce an output that is not reflected + * in state changes, then this item MUST be present after the + * `transaction_result` for that transaction. + */ + com.hedera.hapi.block.stream.output.TransactionOutput transaction_output = 6; - /** - * A set of state changes. - *

- * All changes to values in network state SHALL be described by - * stream items of this type.
- * The source of these state changes SHALL be described by the - * `reason` enumeration. - */ - com.hedera.hapi.block.stream.output.StateChanges state_changes = 7; + /** + * A set of state changes. + *

+ * All changes to values in network state SHALL be described by + * stream items of this type.
+ * The source of these state changes SHALL be described by the + * `reason` enumeration. + */ + com.hedera.hapi.block.stream.output.StateChanges state_changes = 7; - /** - * Verification data for an item filtered from the stream.
- * This is a hash for a merkle tree node where the contents of that - * part of the merkle tree have been removed from this stream. - *

- * Items of this type SHALL NOT be present in the full (unfiltered) - * block stream.
- * Items of this type SHALL replace any item removed from a partial - * (filtered) block stream.
- * Presence of `filtered_item` entries SHALL NOT prevent verification - * of a block, but MAY preclude verification or reconstruction of - * consensus state.
- */ - FilteredItemHash filtered_item_hash = 8; + /** + * Verification data for an item filtered from the stream.
+ * This is a hash for a merkle tree node where the contents of that + * part of the merkle tree have been removed from this stream. + *

+ * Items of this type SHALL NOT be present in the full (unfiltered) + * block stream.
+ * Items of this type SHALL replace any item removed from a partial + * (filtered) block stream.
+ * Presence of `filtered_item` entries SHALL NOT prevent verification + * of a block, but MAY preclude verification or reconstruction of + * consensus state.
+ */ + FilteredItemHash filtered_item_hash = 8; - /** - * A signed block proof.
- * The signed merkle proof for this block. This will validate - * a "virtual" merkle tree containing the previous block "virtual" - * root, an "input" subtree, an "output" subtree, and - * a "state changes" subtree. - *

- * This item is not part of the block stream hash chain/tree, and - * MUST follow after the end of a block. - */ - BlockProof block_proof = 9; + /** + * A signed block proof.
+ * The signed merkle proof for this block. This will validate + * a "virtual" merkle tree containing the previous block "virtual" + * root, an "input" subtree, an "output" subtree, and + * a "state changes" subtree. + *

+ * This item is not part of the block stream hash chain/tree, and + * MUST follow after the end of a block. + */ + BlockProof block_proof = 9; - /** - * A record file and associated data. - *

- * This MUST contain a single Record file, associated Sidecar files, - * and data from related Signature files. - * If this item is present, special treatment is - * REQUIRED for this block. - *

    - *
  • The block SHALL NOT have a `BlockHeader`.
    • - *
  • The block SHALL NOT have a `BlockProof`.
    • - *
  • The block SHALL contain _exactly one_ `RecordFileItem`.
    • - *
  • The block SHALL NOT contain any item other than a - * `RecordFileItem`.
    • - *
  • The content of the `RecordFileItem` MUST be validated using - * the signature data and content provided within according to - * the process used for Record Files prior to the creation - * of Block Stream.
    • - *
- */ - RecordFileItem record_file = 10; + /** + * A record file and associated data. + *

+ * This MUST contain a single Record file, associated Sidecar files, + * and data from related Signature files. + * If this item is present, special treatment is + * REQUIRED for this block. + *

    + *
  • The block SHALL NOT have a `BlockHeader`.
    • + *
  • The block SHALL NOT have a `BlockProof`.
    • + *
  • The block SHALL contain _exactly one_ `RecordFileItem`.
    • + *
  • The block SHALL NOT contain any item other than a + * `RecordFileItem`.
    • + *
  • The content of the `RecordFileItem` MUST be validated using + * the signature data and content provided within according to + * the process used for Record Files prior to the creation + * of Block Stream.
    • + *
+ */ + RecordFileItem record_file = 10; - /** - * A trace data. - *

- * Any informational trace data MAY be described by - * stream items of this type.
- */ - com.hedera.hapi.block.stream.trace.TraceData trace_data = 11; + /** + * A trace data. + *

+ * Any informational trace data MAY be described by + * stream items of this type.
+ */ + com.hedera.hapi.block.stream.trace.TraceData trace_data = 11; - /** - * TODO - */ - FilteredSingleItem filtered_single_item = 20; + /** + * TODO + */ + FilteredSingleItem filtered_single_item = 20; - /** - * TODO - */ - com.hedera.hapi.block.stream.output.BlockFooter block_footer = 21; - } + /** + * TODO + */ + com.hedera.hapi.block.stream.output.BlockFooter block_footer = 21; + } } /** Identifer for each sub-tree of the block root fixed size tree */ diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto index 5b7fbec48e65..188bd9343506 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto @@ -232,7 +232,11 @@ message BlockProof { /** * TODO */ - com.hedera.hapi.block.stream.StateProof state_proof = 8; + oneof proof { + TssSignedBlockProof signed_block_proof = 8; +// com.hedera.hapi.block.stream.StateProof block_state_proof = 9; + SignedRecordFileProof signed_record_file_proof = 10; + } } /** diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto index 60550e1f482a..900a7923e238 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto @@ -35,7 +35,12 @@ message BlockFooter { */ bytes root_hash_of_all_block_hashes_tree = 2; - /** The root hash of the state merkle tree for the version of state at the begining of the current block + /** The root hash of the state merkle tree for the version of state at the beginning of the current block */ bytes start_of_block_state_root_hash = 3; + + /** + * The root hash of the state merkle tree for the version of state at the end of the current block + */ + bytes end_of_block_state_root_hash = 4; } diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto index 6ab1ed2fd551..95923eea2f0d 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto @@ -24,7 +24,8 @@ import "services/state/blockstream/merkle_leaf.proto"; /** * TODO */ -message StateProof { +// For some reason pbj can't figure out the difference between this StateProof and the org.hiero.block.api StateProof +message NoThisStateProof { /** * TODO @@ -40,7 +41,7 @@ message StateProof { /** * TODO */ - SignedRecordFileProof signed_record_file_proof = 3; + SignedRecordFileProof signed_record_file_proof = 4; } } diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto index 928c5b301ea8..3d7e74b5e196 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto @@ -39,139 +39,141 @@ option java_multiple_files = true; * block item in this block. */ message BlockStreamInfo { - /** - * A block number.
- * This is the current block number. - */ - uint64 block_number = 1; - - /** - * A consensus time for the current block.
- * This is the consensus time of the first round in the current block, - * and is used to determine if this block was the first across an - * important boundary in consensus time, such as UTC midnight. - * This may also be used to purge entities expiring between the last - * block time and this time. - */ - proto.Timestamp block_time = 2; - - /** - * A concatenation of hash values.
- * This combines several trailing output block item hashes and - * is used as a seed value for a pseudo-random number generator.
- * This is also required to implement the EVM `PREVRANDAO` opcode.
- * This MUST contain at least 256 bits of entropy. - */ - bytes trailing_output_hashes = 3; - - /** - * A concatenation of hash values.
- * This field combines up to 256 trailing block hashes. - *

- * If this message is for block number N, then the earliest available - * hash SHALL be for block number N-256.
- * The latest available hash SHALL be for block N-1.
- * This is REQUIRED to implement the EVM `BLOCKHASH` opcode. - *

- * ### Field Length - * Each hash value SHALL be the trailing 265 bits of a SHA2-384 hash.
- * The length of this field SHALL be an integer multiple of 32 bytes.
- * This field SHALL be at least 32 bytes.
- * The maximum length of this field SHALL be 8192 bytes. - */ - bytes trailing_block_hashes = 4; - - /** - * A SHA2-384 hash value.
- * This is the hash of the "input" subtree for this block. - */ - bytes input_tree_root_hash = 5; - - /** - * A SHA2-384 hash value.
- * This is the hash of consensus state at the _start_ of this block. - */ - bytes start_of_block_state_hash = 6; - - /** - * A count of "output" block items in this block. - *

- * This SHALL count the number of output block items that _precede_ - * the state change that updates this singleton. - */ - uint32 num_preceding_state_changes_items = 7; - - /** - * TODO - */ - repeated bytes intermediate_previous_block_root_hashes = 8; - - /** - * A block-end consensus time stamp. - *

- * This field SHALL hold the last-used consensus time for - * the current block. - */ - proto.Timestamp block_end_time = 9; - - /** - * Whether the post-upgrade work has been done. - *

- * This MUST be false if and only if the network just restarted - * after an upgrade and has not yet done the post-upgrade work. - */ - bool post_upgrade_work_done = 10; - - /** - * A version describing the version of application software. - *

- * This SHALL be the software version that created this block. - */ - proto.SemanticVersion creation_software_version = 11; - - /** - * The time stamp at which the last interval process was done. - *

- * This field SHALL hold the consensus time for the last time - * at which an interval of time-dependent events were processed. - */ - proto.Timestamp last_interval_process_time = 12; - - /** - * The time stamp at which the last user transaction was handled. - *

- * This field SHALL hold the consensus time for the last time - * at which a user transaction was handled. - */ - proto.Timestamp last_handle_time = 13; - - /** - * TODO - */ - repeated bytes intermediate_consensus_header_hashes = 14; - - /** - * TODO - */ - repeated bytes intermediate_input_block_item_hashes = 15; - - /** - * TODO - */ - repeated bytes intermediate_output_block_item_hashes = 16; - - /** - * TODO - */ - repeated bytes intermediate_state_change_block_item_hashes = 17; - - /** - * TODO - */ - repeated bytes intermediate_trace_data_hashes = 18; - - /** - * TODO - */ - proto.Timestamp block_start_consensus_timestamp = 19; + /** + * A block number.
+ * This is the current block number. + */ + uint64 block_number = 1; + + /** + * A consensus time for the current block.
+ * This is the consensus time of the first round in the current block, + * and is used to determine if this block was the first across an + * important boundary in consensus time, such as UTC midnight. + * This may also be used to purge entities expiring between the last + * block time and this time. + */ + proto.Timestamp block_time = 2; + + /** + * A concatenation of hash values.
+ * This combines several trailing output block item hashes and + * is used as a seed value for a pseudo-random number generator.
+ * This is also required to implement the EVM `PREVRANDAO` opcode.
+ * This MUST contain at least 256 bits of entropy. + */ + bytes trailing_output_hashes = 3; + + /** + * A concatenation of hash values.
+ * This field combines up to 256 trailing block hashes. + *

+ * If this message is for block number N, then the earliest available + * hash SHALL be for block number N-256.
+ * The latest available hash SHALL be for block N-1.
+ * This is REQUIRED to implement the EVM `BLOCKHASH` opcode. + *

+ * ### Field Length + * Each hash value SHALL be the trailing 265 bits of a SHA2-384 hash.
+ * The length of this field SHALL be an integer multiple of 32 bytes.
+ * This field SHALL be at least 32 bytes.
+ * The maximum length of this field SHALL be 8192 bytes. + */ + bytes trailing_block_hashes = 4; + + /** + * A SHA2-384 hash value.
+ * This is the final hash of the "input" subtree for this block. + */ + bytes input_tree_root_hash = 5; + + /** + * A SHA2-384 hash value.
+ * This is the hash of consensus state at the _start_ of this block. + */ + bytes start_of_block_state_hash = 6; + + /** + * A count of "output" block items in this block. + *

+ * This SHALL count the number of output block items that _precede_ + * the state change that updates this singleton. + */ + uint32 num_preceding_state_changes_items = 7; + + /** + * TODO + */ + repeated bytes intermediate_previous_block_root_hashes = 8; + + /** + * A block-end consensus time stamp. + *

+ * This field SHALL hold the last-used consensus time for + * the current block. + */ + proto.Timestamp block_end_time = 9; + + /** + * Whether the post-upgrade work has been done. + *

+ * This MUST be false if and only if the network just restarted + * after an upgrade and has not yet done the post-upgrade work. + */ + bool post_upgrade_work_done = 10; + + /** + * A version describing the version of application software. + *

+ * This SHALL be the software version that created this block. + */ + proto.SemanticVersion creation_software_version = 11; + + /** + * The time stamp at which the last interval process was done. + *

+ * This field SHALL hold the consensus time for the last time + * at which an interval of time-dependent events were processed. + */ + proto.Timestamp last_interval_process_time = 12; + + /** + * The time stamp at which the last user transaction was handled. + *

+ * This field SHALL hold the consensus time for the last time + * at which a user transaction was handled. + */ + proto.Timestamp last_handle_time = 13; + + /** + * A SHA2-384 hash value.
+ * This is the final hash of the "input" subtree for this block. + */ + bytes consensus_header_root_hash = 14; + + /** + * A SHA2-384 hash value.
+ * This is the final hash of the "input" subtree for this block. + */ + bytes output_item_root_hash = 15; + + /** + * A SHA2-384 hash value.
+ * This is the penultimate hash of the "input" subtree for + * this block. The final hash of the "state change" subtree MUST + * be calculated immediately after this block stream info object + * is persisted to state, and its accompanying state change emitted. + */ + bytes penultimate_state_change_item_root_hash = 16; + + /** + * A SHA2-384 hash value.
+ * This is the final hash of the "trace data" subtree for this block. + */ + bytes trace_data_root_hash = 17; + + /** + * TODO + */ + proto.Timestamp block_start_consensus_timestamp = 18; } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 0512db55de49..1c0d96686f55 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -25,6 +25,8 @@ import com.hedera.hapi.block.stream.BlockProof; import com.hedera.hapi.block.stream.ChainOfTrustProof; import com.hedera.hapi.block.stream.MerkleSiblingHash; +import com.hedera.hapi.block.stream.SubMerkleTree; +import com.hedera.hapi.block.stream.TssSignedBlockProof; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.block.stream.output.SingletonUpdateChange; import com.hedera.hapi.block.stream.output.StateChange; @@ -38,6 +40,7 @@ import com.hedera.node.app.blocks.BlockStreamManager; import com.hedera.node.app.blocks.BlockStreamService; import com.hedera.node.app.blocks.InitialStateHash; +import com.hedera.node.app.blocks.StreamingTreeHasher; import com.hedera.node.app.hapi.utils.CommonUtils; import com.hedera.node.app.info.DiskStartupNetworks; import com.hedera.node.app.info.DiskStartupNetworks.InfoType; @@ -74,6 +77,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -145,11 +149,17 @@ public class BlockStreamManagerImpl implements BlockStreamManager { // block merkle tree private IncrementalStreamingHasher previousBlockHashes; // ALL previous hashes, but streaming-collapsed private Bytes stateHashAtStartOfBlock; - private IncrementalStreamingHasher consensusHeaderHasher; - private IncrementalStreamingHasher inputTreeHasher; - private IncrementalStreamingHasher outputTreeHasher; - private IncrementalStreamingHasher stateChangesHasher; - private IncrementalStreamingHasher traceDataHasher; + // private IncrementalStreamingHasher consensusHeaderHasher; + // private IncrementalStreamingHasher inputTreeHasher; + // private IncrementalStreamingHasher outputTreeHasher; + // private IncrementalStreamingHasher stateChangesHasher; + // private IncrementalStreamingHasher traceDataHasher; + + private StreamingTreeHasher consensusHeaderHasher; + private StreamingTreeHasher inputTreeHasher; + private StreamingTreeHasher outputTreeHasher; + private StreamingTreeHasher stateChangesHasher; + private StreamingTreeHasher traceDataHasher; // end block merkle tree private BlockStreamManagerTask worker; @@ -288,87 +298,80 @@ public void initBlockTrees(@NonNull final State state, @Nullable final Bytes las final var prevBlocksHasher = blockStreamInfo.intermediatePreviousBlockRootHashes().stream() .map(Bytes::toByteArray) .toList(); - final var prevBlocksHash = Bytes.wrap(new IncrementalStreamingHasher(prevBlocksHasher).computeRootHash()); - // Branch 3: - final var blockStartStateHash = blockStreamInfo.startOfBlockStateHash(); + previousBlockHashes = new IncrementalStreamingHasher(prevBlocksHasher); + final var allPrevBlocksHash = Bytes.wrap(new IncrementalStreamingHasher(prevBlocksHasher).computeRootHash()); + // Branch 3 + // Retrieve the previous block's starting state hash (not done right here, just part of the calculated last + // block hash below) // Branch 4 - final var consensusHeaders = blockStreamInfo.intermediateConsensusHeaderHashes().stream() - .map(Bytes::toByteArray) - .toList(); - final var consensusHeadersHash = Bytes.wrap(new IncrementalStreamingHasher(consensusHeaders).computeRootHash()); + consensusHeaderHasher = new NaiveStreamingTreeHasher(); // Branch 5 - final var inputItems = blockStreamInfo.intermediateInputBlockItemHashes().stream() - .map(Bytes::toByteArray) - .toList(); - final var inputsHash = Bytes.wrap(new IncrementalStreamingHasher(inputItems).computeRootHash()); + inputTreeHasher = new NaiveStreamingTreeHasher(); // Branch 6 - final var outputItems = blockStreamInfo.intermediateOutputBlockItemHashes().stream() - .map(Bytes::toByteArray) - .toList(); - final var outputsHash = Bytes.wrap(new IncrementalStreamingHasher(outputItems).computeRootHash()); - // Branch 7, the state changes hash, will come immediately following + outputTreeHasher = new NaiveStreamingTreeHasher(); + // Branch 7 + stateChangesHasher = new NaiveStreamingTreeHasher(); // Branch 8 - final var traceItems = blockStreamInfo.intermediateTraceDataHashes().stream() - .map(Bytes::toByteArray) - .toList(); - final var traceDataHash = Bytes.wrap(new IncrementalStreamingHasher(traceItems).computeRootHash()); + traceDataHasher = new NaiveStreamingTreeHasher(); - // The final ingredient, the state changes tree root hash (branch 7), is not directly in the BlockStreamInfo, - // but we can recompute it based on the tree hash information and the fact the last state changes item in the - // block was devoted to putting the BlockStreamInfo itself into the state - // First, construct the final state change and add the final state change as a leaf - stateChangesHasher = stateChangesSubTreeRootHashFrom(blockStreamInfo); - // Compute the state change subtree's current root hash - final var stateChangesHash = Bytes.wrap(stateChangesHasher.computeRootHash()); - - // Now, define the remaining subtrees - previousBlockHashes = new IncrementalStreamingHasher(prevBlocksHasher); - consensusHeaderHasher = new IncrementalStreamingHasher(consensusHeaders); - inputTreeHasher = new IncrementalStreamingHasher(inputItems); - outputTreeHasher = new IncrementalStreamingHasher(outputItems); - traceDataHasher = new IncrementalStreamingHasher(traceItems); + // We have to calculate the final hash of the previous block's state changes subtree because only the + // penultimate state hash is in the block stream info object + final var lastBlockPenultimateStateHash = blockStreamInfo.penultimateStateChangeItemRootHash(); + // Reconstruct the final state change block item that would have been emitted + final var lastBlockFinalStateChange = StateChange.newBuilder() + .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal()) + .singletonUpdate(SingletonUpdateChange.newBuilder() + .blockStreamInfoValue(blockStreamInfo) + .build()) + .build(); + final var changeBytes = StateChange.PROTOBUF.toBytes(lastBlockFinalStateChange); + // Combine the penultimate hash and the hash of the reconstructed state change item to produce the previous + // block's final state changes hash + final var lastBlockFinalStateChangesHash = BlockImplUtils.combine(lastBlockPenultimateStateHash, changeBytes); final var calculatedLastBlockHash = Optional.ofNullable(lastBlockHash) .orElseGet(() -> BlockStreamManagerImpl.combine( prevBlockHash, - prevBlocksHash, - blockStartStateHash, - consensusHeadersHash, - inputsHash, - outputsHash, - stateChangesHash, - traceDataHash, + allPrevBlocksHash, + blockStreamInfo.startOfBlockStateHash(), + blockStreamInfo.consensusHeaderRootHash(), + blockStreamInfo.inputTreeRootHash(), + blockStreamInfo.outputItemRootHash(), + lastBlockFinalStateChangesHash, + blockStreamInfo.traceDataRootHash(), blockStreamInfo.blockStartConsensusTimestamp())); requireNonNull(calculatedLastBlockHash); initLastBlockHash(calculatedLastBlockHash); } - /** - * Given a {@link BlockStreamInfo} context, computes the state changes tree root hash that must have been - * computed at the end of the block that the context describes, assuming the final state change block item - * was the state change that put the context into the state. - * - * @param info the context to use - * @return the inferred output tree root hash - */ - private @NonNull IncrementalStreamingHasher stateChangesSubTreeRootHashFrom(@NonNull final BlockStreamInfo info) { - // Construct the final state change - final var blockStreamInfoChange = StateChange.newBuilder() - .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal()) - .singletonUpdate(SingletonUpdateChange.newBuilder() - .blockStreamInfoValue(info) - .build()) - .build(); - final var changeBytes = StateChange.PROTOBUF.toBytes(blockStreamInfoChange); - - // Add the final state change as a leaf and compute the root - final var stateChangeSubTree = - new IncrementalStreamingHasher(info.intermediateStateChangeBlockItemHashes().stream() - .map(Bytes::toByteArray) - .toList()); - stateChangeSubTree.addLeaf(changeBytes.toByteArray()); - return stateChangeSubTree; - } + // /** + // * Given a {@link BlockStreamInfo} context, computes the state changes tree root hash that must have been + // * computed at the end of the block that the context describes, assuming the final state change block item + // * was the state change that put the context into the state. + // * + // * @param info the context to use + // * @return the inferred output tree root hash + // */ + // private @NonNull StreamingTreeHasher stateChangesSubTreeRootHashFrom(@NonNull final BlockStreamInfo info) { + // // Construct the final state change + // final var blockStreamInfoChange = StateChange.newBuilder() + // .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal()) + // .singletonUpdate(SingletonUpdateChange.newBuilder() + // .blockStreamInfoValue(info) + // .build()) + // .build(); + // final var changeBytes = StateChange.PROTOBUF.toBytes(blockStreamInfoChange); + // + // // Combine the block info's last state change–the penultimate state change of the previous block—with the + // reconstructed final state change to get the new starting state root hash + // final var blockStartingStateChangesRoot = BlockImplUtils.combine(info.stateChangeBlockItemRootHash(), + // changeBytes); + // + // // Construct the new state changes hasher, using the block starting state changes root hash as the starting hash + // final var stateChangeSubTree = new NaiveStreamingTreeHasher(); + // stateChangeSubTree.addLeaf(ByteBuffer.wrap(blockStartingStateChangesRoot.toByteArray())); + // return stateChangeSubTree; + // } @Override public void startRound(@NonNull final Round round, @NonNull final State state) { @@ -621,70 +624,98 @@ public boolean endRound(@NonNull final State state, final long roundNum) { // Branch 2 final var prevBlockRootsHash = Bytes.wrap(previousBlockHashes.computeRootHash()); // Branch 3: blockStartStateHash - // Branch 4 - final var consensusHeaderHash = Bytes.wrap(consensusHeaderHasher.computeRootHash()); - // Branch 5 - final var inputsHash = Bytes.wrap(inputTreeHasher.computeRootHash()); - // Branch 6 - final var outputsHash = Bytes.wrap(outputTreeHasher.computeRootHash()); - // Branch 7 - final var stateChangesHash = Bytes.wrap(stateChangesHasher.computeRootHash()); - // Branch 8 - final var traceDataHash = Bytes.wrap(traceDataHasher.computeRootHash()); + + // Calculate hashes for branches 4-8 + final Map computedHashes = new HashMap<>(); + final var future = CompletableFuture.allOf( + // Branch 4 + consensusHeaderHasher + .rootHash() + .thenAccept(b -> computedHashes.put(SubMerkleTree.CONSENSUS_HEADER_ITEMS, b)), + // Branch 5 + inputTreeHasher.rootHash().thenAccept(b -> computedHashes.put(SubMerkleTree.INPUT_ITEMS_TREE, b)), + // Branch 6 + outputTreeHasher.rootHash().thenAccept(b -> computedHashes.put(SubMerkleTree.OUTPUT_ITEMS_TREE, b)), + // Branch 7 + stateChangesHasher + .rootHash() + .thenAccept(b -> computedHashes.put(SubMerkleTree.STATE_CHANGE_ITEMS_TREE, b)), + // Branch 8 + traceDataHasher + .rootHash() + .thenAccept(b -> computedHashes.put(SubMerkleTree.TRACE_DATA_ITEMS_TREE, b))); + future.join(); + + // Branch 4 final hash: + final var consensusHeaderHash = computedHashes.get(SubMerkleTree.CONSENSUS_HEADER_ITEMS); + // Branch 5 final hash: + final var inputsHash = computedHashes.get(SubMerkleTree.INPUT_ITEMS_TREE); + // Branch 6 final hash: + final var outputsHash = computedHashes.get(SubMerkleTree.OUTPUT_ITEMS_TREE); + // Branch 7 (penultimate because there will be one more state change when the block stream info object is + // stored) + final var penultimateStateChangesHash = computedHashes.get(SubMerkleTree.STATE_CHANGE_ITEMS_TREE); + // Branch 8 final hash: + final var traceDataHash = computedHashes.get(SubMerkleTree.TRACE_DATA_ITEMS_TREE); // Put this block hash context in state via the block stream info final var writableState = state.getWritableStates(BlockStreamService.NAME); final var blockStreamInfoState = writableState.getSingleton(BLOCK_STREAM_INFO_STATE_ID); - blockStreamInfoState.put(new BlockStreamInfo( + final var newBlockStreamInfo = new BlockStreamInfo( blockNumber, blockTimestamp(), runningHashManager.latestHashes(), // lastBlockHash is stored here blockHashManager.blockHashes(), inputsHash, blockStartStateHash, - stateChangesHasher.leafCount(), + stateChangesHasher.status().numLeaves(), previousBlockHashes.intermediateHashingState(), lastUsedTime, pendingWork != POST_UPGRADE_WORK, version, asTimestamp(lastIntervalProcessTime), asTimestamp(lastTopLevelTime), - consensusHeaderHasher.intermediateHashingState(), - inputTreeHasher.intermediateHashingState(), - outputTreeHasher.intermediateHashingState(), - stateChangesHasher.intermediateHashingState(), - traceDataHasher.intermediateHashingState(), - asTimestamp(firstConsensusTimeOfCurrentBlock))); + consensusHeaderHash, + outputsHash, + penultimateStateChangesHash, + traceDataHash, + asTimestamp(firstConsensusTimeOfCurrentBlock)); + blockStreamInfoState.put(newBlockStreamInfo); ((CommittableWritableStates) writableState).commit(); // Produce one more state change item (i.e. putting the block stream info just constructed into state) worker.addItem(flushChangesFromListener(boundaryStateChangeListener)); worker.sync(); - final var blockHash = combine( - lastBlockHash, - prevBlockRootsHash, - stateHashAtStartOfBlock, - consensusHeaderHash, - inputsHash, - outputsHash, - stateChangesHash, - traceDataHash, - firstConsensusTimeOfCurrentBlock); - - // Compute depth two hashes - final var depth2Node0 = combine(lastBlockHash, blockStartStateHash); - final var depth2Node1 = combine(consensusHeaderHash, inputHash); - final var depth2Node2 = combine(outputHash, stateChangesHash); - final var depth2Node3 = combine(traceDataHash, NULL_HASH); + // Reconstruct the final state change in order to calculate the final state change subtree hash + final var blockStreamInfoChange = StateChange.newBuilder() + .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal()) + .singletonUpdate(SingletonUpdateChange.newBuilder() + .blockStreamInfoValue(newBlockStreamInfo) + .build()) + .build(); + final var changeBytes = StateChange.PROTOBUF.toBytes(blockStreamInfoChange); + // Combine the penultimate state change leaf with the final state change leaf + final var finalStateChangesHash = BlockImplUtils.combine(penultimateStateChangesHash, changeBytes); + + final var finalCurrentBlockHash = combine( + lastBlockHash, + prevBlockRootsHash, + stateHashAtStartOfBlock, + consensusHeaderHash, + inputsHash, + outputsHash, + finalStateChangesHash, + traceDataHash, + firstConsensusTimeOfCurrentBlock); // Create BlockFooter with the three essential hashes: // 1. previousBlockRootHash - Root hash of the previous block (N-1) - // 2. rootHashOfAllBlockHashesTree - Streaming tree of all block hashes 0..N-1 + // 2. rootHashOfAllBlockHashesTree - RootStreaming tree of all block hashes 0..N-1 // 3. startOfBlockStateRootHash - State hash at the beginning of current block final var blockFooter = com.hedera.hapi.block.stream.output.BlockFooter.newBuilder() .previousBlockRootHash(lastBlockHash) - .rootHashOfAllBlockHashesTree(blockHash) + .rootHashOfAllBlockHashesTree(finalCurrentBlockHash) .startOfBlockStateRootHash(blockStartStateHash) .build(); @@ -694,20 +725,12 @@ public boolean endRound(@NonNull final State state, final long roundNum) { worker.addItem(footerItem); worker.sync(); - // TODO: construct pending block proof - final var pendingProof = BlockProof.newBuilder(); - // pendingBlocks.add(new PendingBlock( - // blockNumber, - // null, - // blockHash, - // pendingProof, - // writer, - // new MerkleSiblingHash(false, blockStartStateHash), - // new MerkleSiblingHash(false, depth2Node1), - // new MerkleSiblingHash(false, depth1Node1))); + // Create a pending block, waiting to be signed + final var blockProofBuilder = BlockProof.newBuilder(); + pendingBlocks.add(new PendingBlock(blockNumber, null, finalCurrentBlockHash, blockProofBuilder, writer)); // Update in-memory state to prepare for the next block - lastBlockHash = blockHash; + lastBlockHash = finalCurrentBlockHash; writer = null; // Special case when signing with hinTS and this is the freeze round; we have to wait @@ -717,10 +740,13 @@ public boolean endRound(@NonNull final State state, final long roundNum) { // In case the id of the next hinTS construction changed since a block ended pendingBlocks.forEach(block -> block.flushPending(hasPrecedingUnproven.getAndSet(true))); } else { - final var attempt = blockHashSigner.sign(blockHash); + final var attempt = blockHashSigner.sign(finalCurrentBlockHash); attempt.signatureFuture() .thenAcceptAsync(signature -> finishProofWithSignature( - blockHash, signature, attempt.verificationKey(), attempt.chainOfTrustProof())); + finalCurrentBlockHash, + signature, + attempt.verificationKey(), + attempt.chainOfTrustProof())); } final var exportNetworkToDisk = @@ -751,8 +777,6 @@ public boolean endRound(@NonNull final State state, final long roundNum) { writer = null; } requireNonNull(fatalShutdownFuture).complete(null); - - // TODO: write intermediate hashes of sub trees? } return closesBlock; } @@ -839,9 +863,11 @@ private synchronized void finishProofWithSignature( if (!siblingHashes.isEmpty()) { indirectProofCounter.increment(); } + // TODO: (possibly) construct state proof when state proof's protobuf definition is finalized? final var proof = block.proofBuilder() - .blockSignature(blockSignature) - .siblingHashes(siblingHashes.stream().flatMap(List::stream).toList()); + .signedBlockProof(TssSignedBlockProof.newBuilder() + .blockSignature(blockSignature) + .build()); if (verificationKey != null) { proof.verificationKey(verificationKey); if (chainOfTrustProof != null) { @@ -999,15 +1025,15 @@ class SequentialTask extends AbstractTask { protected boolean onExecute() { final var kind = item.item().kind(); switch (kind) { - case ROUND_HEADER, EVENT_HEADER -> consensusHeaderHasher.addLeaf(hash.array()); - case SIGNED_TRANSACTION -> inputTreeHasher.addLeaf(hash.array()); + case ROUND_HEADER, EVENT_HEADER -> consensusHeaderHasher.addLeaf(hash); + case SIGNED_TRANSACTION -> inputTreeHasher.addLeaf(hash); case TRANSACTION_RESULT -> { runningHashManager.nextResultHash(hash); hash.rewind(); - outputTreeHasher.addLeaf(hash.array()); + outputTreeHasher.addLeaf(hash); } - case TRANSACTION_OUTPUT, BLOCK_HEADER -> outputTreeHasher.addLeaf(hash.array()); - case STATE_CHANGES -> stateChangesHasher.addLeaf(hash.array()); + case TRANSACTION_OUTPUT, BLOCK_HEADER -> outputTreeHasher.addLeaf(hash); + case STATE_CHANGES -> stateChangesHasher.addLeaf(hash); case BLOCK_FOOTER, BLOCK_PROOF -> { // BlockFooter and BlockProof are not included in any merkle tree // They are metadata about the block, not part of the hashed content diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index 6ec4a914994d..66e70d4f4509 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -75,8 +75,6 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java index 9f6a6058c19c..c17559045fca 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java @@ -8,6 +8,8 @@ import static com.hedera.services.bdd.spec.utilops.BlockNodeVerbs.blockNode; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.assertBlockNodeCommsLogContainsTimeframe; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.assertBlockNodeCommsLogDoesNotContain; +import static com.hedera.services.bdd.spec.utilops.UtilVerbs.assertHgcaaLogContainsTimeframe; +import static com.hedera.services.bdd.spec.utilops.UtilVerbs.assertHgcaaLogDoesNotContain; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.doingContextual; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.sourcingContextual; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.waitForActive; From eb2912d305ab3c3256ea46c9c85d14220ae33a08 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Thu, 23 Oct 2025 02:01:01 -0600 Subject: [PATCH 25/63] NoThisStateProof block proof Signed-off-by: Matt Hess --- .../src/main/proto/block/stream/block_proof.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto index 188bd9343506..b324ac456549 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto @@ -234,7 +234,7 @@ message BlockProof { */ oneof proof { TssSignedBlockProof signed_block_proof = 8; -// com.hedera.hapi.block.stream.StateProof block_state_proof = 9; + com.hedera.hapi.block.stream.NoThisStateProof block_state_proof = 9; SignedRecordFileProof signed_record_file_proof = 10; } } From 7433e8c77e861503b8f4d9e0453864f4bab2b673 Mon Sep 17 00:00:00 2001 From: Edward Wertz Date: Fri, 24 Oct 2025 14:03:40 -0500 Subject: [PATCH 26/63] gradle work around to compile `message StateProof` Signed-off-by: Edward Wertz --- hapi/hapi/build.gradle.kts | 4 ++-- .../src/main/proto/block/stream/block_proof.proto | 2 +- .../src/main/proto/block/stream/state_proof.proto | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hapi/hapi/build.gradle.kts b/hapi/hapi/build.gradle.kts index 365d6b47d785..f07721c7ed10 100644 --- a/hapi/hapi/build.gradle.kts +++ b/hapi/hapi/build.gradle.kts @@ -30,13 +30,13 @@ sourceSets { pbj { srcDir(protoApiSrc) srcDir(tasks.extractProto) // see comment on the 'dependencies' block - exclude("mirror", "sdk", "internal") + exclude("mirror", "sdk", "internal", "block-node/api/proof_service.proto") } // The below should be replaced with a 'requires com.hedera.protobuf.java.api' // in testFixtures scope - #14026 proto { srcDir(protoApiSrc) - exclude("mirror", "sdk", "internal") + exclude("mirror", "sdk", "internal", "block-node/api/proof_service.proto") } } } diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto index b324ac456549..75a18a1577b1 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto @@ -234,7 +234,7 @@ message BlockProof { */ oneof proof { TssSignedBlockProof signed_block_proof = 8; - com.hedera.hapi.block.stream.NoThisStateProof block_state_proof = 9; + StateProof block_state_proof = 9; SignedRecordFileProof signed_record_file_proof = 10; } } diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto index 95923eea2f0d..1f3a8bdd4b11 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto @@ -25,7 +25,7 @@ import "services/state/blockstream/merkle_leaf.proto"; * TODO */ // For some reason pbj can't figure out the difference between this StateProof and the org.hiero.block.api StateProof -message NoThisStateProof { +message StateProof { /** * TODO From aee237ec67982a03ef988daaf4621e5e1393a1ad Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Thu, 23 Oct 2025 08:49:13 -0600 Subject: [PATCH 27/63] Remove unneeded properties Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_proof.proto | 135 +----------------- .../block/stream/output/block_footer.proto | 5 - 2 files changed, 2 insertions(+), 138 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto index 75a18a1577b1..5d6c1b69cd73 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto @@ -86,148 +86,17 @@ message BlockProof { */ uint64 block = 1; - /** - * A block root hash for the previous block. - *

- * This value MUST match the block merkle tree root hash of the previous - * block in the block stream.
- * This value SHALL be empty for the genesis block, and SHALL NOT be empty - * for any other block.
- * Client systems SHOULD optimistically reject any block with a - * `previous_block_proof_hash` that does not match the block hash of the - * previous block and MAY assume the block stream has encountered data - * loss, data corruption, or unauthorized modification. - *

- * The process for computing a block hash is somewhat complex, and involves - * creating a "virtual" merkle tree to obtain the root merkle hash of - * that virtual tree.
- * The merkle tree SHALL have a 4 part structure with 2 internal nodes, - * structured in a strictly binary tree. - *

    - *
  • The merkle tree root SHALL be the parent of both - * internal nodes. - *
      - *
    1. The first "internal" node SHALL be the parent of the - * two "left-most" nodes. - *
        - *
      1. The first leaf MUST be the previous block hash, and is a - * single 48-byte value.
        2. - *
      2. The second leaf MUST be the root of a, strictly binary, - * merkle tree composed of all "input" block items in - * the block.
        - * Input items SHALL be transactions, system transactions, - * and events.
        - * Leaf nodes in this subtree SHALL be ordered in the - * same order that the block items are encountered - * in the stream.
        3. - *
      - *
      2. - *
    2. The second "internal" node SHALL be the parent of the - * two "right-most" nodes. - *
        - *
      1. The third leaf MUST be the root of a, strictly binary, - * merkle tree composed of all "output" block items in - * the block.
        - * Output items SHALL be transaction result, transaction - * output, and state changes.
        - * Leaf nodes in this subtree SHALL be ordered in the - * same order that the block items are encountered - * in the stream.
        2. - *
      2. The fourth leaf MUST be the merkle tree root hash for - * network state at the start of the block, and is a single - * 48-byte value.
        3. - *
      - *
      3. - *
    - *
    • - *
  • The block hash SHALL be the SHA-384 hash calculated for the root - * of this merkle tree.
    • - *
- */ - bytes previous_block_root_hash = 2 [deprecated = true]; - - /** - * A merkle root hash of the network state.
- * This is present to support validation of this block proof by clients - * that do not maintain a full copy of the network state. - *

- * This MUST contain a hash of the "state" merkle tree root at the start - * of the current block (which this block proof verifies).
- * State processing clients SHOULD calculate the state root hash - * independently and SHOULD NOT rely on this value.
- * State processing clients MUST validate the application of state changes - * for a block using the value present in the Block Proof of the - * _following_ block. - * Compliant consensus nodes MUST produce an "empty" block (containing - * only `BlockHeader` and `BlockProof` as the last block prior to a - * network "freeze" to ensure the final state hash is incorporated into - * the Block Stream correctly. - * Stateless (non-state-processing) clients MUST use this value to - * construct the block merkle tree. - */ - bytes start_of_block_state_root_hash = 3 [deprecated = true]; - - /** - * A TSS signature for one block.
- * This is a single signature representing the collection of partial - * signatures from nodes holding strictly greater than 2/3 of the - * current network "weight" in aggregate. The signature is produced by - * cryptographic "aggregation" of the partial signatures to produce a - * single signature that can be verified with the network public key, - * but could not be produced by fewer nodes than required to meet the - * threshold for network stake "weight". - *

- * This message MUST make use of a threshold signature scheme like `BLS` - * which provides the necessary cryptographic guarantees.
- * This signature SHALL use a TSS signature to provide a single signature - * that represents the consensus signature of consensus nodes.
- * The exact subset of nodes that signed SHALL neither be known nor - * tracked, but it SHALL be cryptographically verifiable that the - * threshold was met if the signature itself can be validated with - * the network public key (a.k.a `LedgerID`). - */ - bytes block_signature = 4 [deprecated = true]; - - /** - * A set of hash values along with ordering information.
- * This list of hash values form the set of sibling hash values needed to - * correctly reconstruct the parent hash, and all hash values "above" that - * hash in the merkle tree. - *

- * A Block proof can be constructed by combining the sibling hashes for - * a previous block hash and sibling hashes for each entry "above" that - * node in the merkle tree of a block proof that incorporates that previous - * block hash. This form of block proof may be used to prove a chain of - * blocks when one or more older blocks is missing the original block - * proof that signed the block's merkle root directly. - *

- * This list MUST be ordered from the sibling of the node that contains - * this block's root node hash, and continues up the merkle tree to the - * root hash of the signed block proof. - *

- * If this block proof has a "direct" signature, then this list MUST be - * empty.
- * If this list is not empty, then this block proof MUST be verified by - * first constructing the "block" merkle tree and computing the root hash - * of that tree, then combining that hash with the values in this list, - * paying attention to the first/second sibling ordering, until the root - * merkle hash is produced from the last pair of sibling hashes. That - * "secondary" root hash MUST then be verified using - * the value of `block_signature`. - */ - repeated MerkleSiblingHash sibling_hashes = 5 [deprecated = true]; - /** * The hinTS key that this signature verifies under; a stream consumer should * only use this key after first checking the chain of trust proof. */ - bytes verification_key = 6; + bytes verification_key = 2; /** * Proof the hinTS verification key is in the chain of trust extending * from the network's ledger id. */ - ChainOfTrustProof verification_key_proof = 7; + ChainOfTrustProof verification_key_proof = 3; /** * TODO diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto index 900a7923e238..76a36f1cf0d4 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto @@ -38,9 +38,4 @@ message BlockFooter { /** The root hash of the state merkle tree for the version of state at the beginning of the current block */ bytes start_of_block_state_root_hash = 3; - - /** - * The root hash of the state merkle tree for the version of state at the end of the current block - */ - bytes end_of_block_state_root_hash = 4; } From 2365bf0624b2431daa8b4641a697c1e09da5a7d8 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Fri, 24 Oct 2025 15:18:11 -0600 Subject: [PATCH 28/63] Protobuf tweaks Signed-off-by: Matt Hess --- .../main/proto/network/pending_proof.proto | 2 +- .../main/proto/block/stream/block_item.proto | 38 +----------------- .../main/proto/block/stream/block_proof.proto | 39 ++++++++++++++++--- .../main/proto/block/stream/state_proof.proto | 1 - 4 files changed, 36 insertions(+), 44 deletions(-) diff --git a/hapi/hapi/src/main/proto/network/pending_proof.proto b/hapi/hapi/src/main/proto/network/pending_proof.proto index d06c9d3d15a6..442f996ebf38 100644 --- a/hapi/hapi/src/main/proto/network/pending_proof.proto +++ b/hapi/hapi/src/main/proto/network/pending_proof.proto @@ -30,7 +30,7 @@ message PendingProof { */ uint64 block = 1; /** - * The hash requiring a TSS signature. + * The hash requiring a proof. */ bytes block_hash = 2; /** diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index acddc7ff03a7..3806ef2b51ec 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -38,6 +38,7 @@ import "block/stream/output/transaction_output.proto"; import "block/stream/output/transaction_result.proto"; import "block/stream/trace/trace_data.proto"; import "block/stream/output/block_footer.proto"; +import "services/state/blockstream/streaming_tree_snapshot.proto"; /** * A single item within a block stream. @@ -274,27 +275,6 @@ message BlockItem { } } -/** Identifer for each sub-tree of the block root fixed size tree */ -enum SubMerkleTree { - ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice - PREVIOUS_BLOCK_ROOT = 1; - PREVIOUS_ROOTS_TREE = 2; - PREVIOUS_BLOCK_START_STATE = 3; - CONSENSUS_HEADER_ITEMS = 4; - INPUT_ITEMS_TREE = 5; - OUTPUT_ITEMS_TREE = 6; - STATE_CHANGE_ITEMS_TREE = 7; - TRACE_DATA_ITEMS_TREE = 8; - FUTURE_1 = 9; // these place holders for future use sub trees, will be renamed if they are used later - FUTURE_2 = 10; - FUTURE_3 = 11; - FUTURE_4 = 12; - FUTURE_5 = 13; - FUTURE_6 = 14; - FUTURE_7 = 15; - FUTURE_8 = 16; -} - /** * Verification data for an item filtered from the stream. * @@ -403,19 +383,3 @@ message RedactedItem { */ SubMerkleTree tree = 3; } - -/** - * TODO - */ -message StreamingTreeSnapshot { - - /** - * Which of the block merkle sub trees this snapshot represents - */ - SubMerkleTree type = 1; - - /** - * All the uncollapsed nodes of the sub tree - */ - repeated bytes nodes = 2; -} diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto index 5d6c1b69cd73..8152f5076373 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto @@ -86,25 +86,54 @@ message BlockProof { */ uint64 block = 1; + /** + * A set of hash values along with ordering information.
+ * This list of hash values form the set of sibling hash values needed to + * correctly reconstruct the parent hash, and all hash values "above" that + * hash in the merkle tree. + *

+ * A Block proof can be constructed by combining the sibling hashes for + * a previous block hash and sibling hashes for each entry "above" that + * node in the merkle tree of a block proof that incorporates that previous + * block hash. This form of block proof may be used to prove a chain of + * blocks when one or more older blocks is missing the original block + * proof that signed the block's merkle root directly. + *

+ * This list MUST be ordered from the sibling of the node that contains + * this block's root node hash, and continues up the merkle tree to the + * root hash of the signed block proof. + *

+ * If this block proof has a "direct" signature, then this list MUST be + * empty.
+ * If this list is not empty, then this block proof MUST be verified by + * first constructing the "block" merkle tree and computing the root hash + * of that tree, then combining that hash with the values in this list, + * paying attention to the first/second sibling ordering, until the root + * merkle hash is produced from the last pair of sibling hashes. That + * "secondary" root hash MUST then be verified using + * the value of `block_signature`. + */ + repeated MerkleSiblingHash sibling_hashes = 2; + /** * The hinTS key that this signature verifies under; a stream consumer should * only use this key after first checking the chain of trust proof. */ - bytes verification_key = 2; + bytes verification_key = 3; /** * Proof the hinTS verification key is in the chain of trust extending * from the network's ledger id. */ - ChainOfTrustProof verification_key_proof = 3; + ChainOfTrustProof verification_key_proof = 4; /** * TODO */ oneof proof { - TssSignedBlockProof signed_block_proof = 8; - StateProof block_state_proof = 9; - SignedRecordFileProof signed_record_file_proof = 10; + TssSignedBlockProof signed_block_proof = 5; + StateProof block_state_proof = 6; + SignedRecordFileProof signed_record_file_proof = 7; } } diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto index 1f3a8bdd4b11..5ec8e17d3081 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto @@ -24,7 +24,6 @@ import "services/state/blockstream/merkle_leaf.proto"; /** * TODO */ -// For some reason pbj can't figure out the difference between this StateProof and the org.hiero.block.api StateProof message StateProof { /** From 7dfb4dad6155e9063f05f33dc170d574667d5c36 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Fri, 24 Oct 2025 15:21:57 -0600 Subject: [PATCH 29/63] Protobuf tweaks Signed-off-by: Matt Hess --- .../blockstream/streaming_tree_snapshot.proto | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/streaming_tree_snapshot.proto diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/streaming_tree_snapshot.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/streaming_tree_snapshot.proto new file mode 100644 index 000000000000..d4501e184a1b --- /dev/null +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/streaming_tree_snapshot.proto @@ -0,0 +1,55 @@ +/** + * TODO + * + * ### Keywords + * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", + * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this + * document are to be interpreted as described in + * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in + * [RFC8174](https://www.ietf.org/rfc/rfc8174). + */ +syntax = "proto3"; + +package com.hedera.hapi.block.stream; + +// SPDX-License-Identifier: Apache-2.0 +option java_package = "com.hedera.hapi.block.stream.protoc"; +// <<>> This comment is special code for setting PBJ Compiler java package +option java_multiple_files = true; + +/** + * TODO + */ +message StreamingTreeSnapshot { + + /** + * Which of the block merkle sub trees this snapshot represents + */ + SubMerkleTree type = 1; + + /** + * All the uncollapsed nodes of the sub tree + */ + repeated bytes nodes = 2; +} + +/** Identifer for each sub-tree of the block root fixed size tree */ +enum SubMerkleTree { + ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice + PREVIOUS_BLOCK_ROOT = 1; + PREVIOUS_ROOTS_TREE = 2; + PREVIOUS_BLOCK_START_STATE = 3; + CONSENSUS_HEADER_ITEMS = 4; + INPUT_ITEMS_TREE = 5; + OUTPUT_ITEMS_TREE = 6; + STATE_CHANGE_ITEMS_TREE = 7; + TRACE_DATA_ITEMS_TREE = 8; + FUTURE_1 = 9; // these place holders for future use sub trees, will be renamed if they are used later + FUTURE_2 = 10; + FUTURE_3 = 11; + FUTURE_4 = 12; + FUTURE_5 = 13; + FUTURE_6 = 14; + FUTURE_7 = 15; + FUTURE_8 = 16; +} \ No newline at end of file From cedab7ee6c7c601058b6950feddd9dbe2f78f316 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Fri, 24 Oct 2025 15:31:26 -0600 Subject: [PATCH 30/63] Proof mechanism Signed-off-by: Matt Hess --- .../blocks/impl/BlockStreamManagerImpl.java | 218 ++++++++---------- .../blocks/impl/IncrementalHasherStorage.java | 4 +- .../impl/streaming/FileBlockItemWriter.java | 5 +- .../impl/BlockStreamManagerImplTest.java | 12 +- .../blocks/impl/streaming/BlockTestUtils.java | 5 +- .../streaming/GrpcBlockItemWriterTest.java | 10 +- .../block/StateChangesValidator.java | 152 ++++++------ 7 files changed, 195 insertions(+), 211 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 1c0d96686f55..cba6edd07404 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -24,8 +24,9 @@ import com.hedera.hapi.block.stream.BlockItem; import com.hedera.hapi.block.stream.BlockProof; import com.hedera.hapi.block.stream.ChainOfTrustProof; +import com.hedera.hapi.block.stream.MerklePath; import com.hedera.hapi.block.stream.MerkleSiblingHash; -import com.hedera.hapi.block.stream.SubMerkleTree; +import com.hedera.hapi.block.stream.StateProof; import com.hedera.hapi.block.stream.TssSignedBlockProof; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.block.stream.output.SingletonUpdateChange; @@ -34,6 +35,7 @@ import com.hedera.hapi.node.base.SemanticVersion; import com.hedera.hapi.node.base.Timestamp; import com.hedera.hapi.node.state.blockstream.BlockStreamInfo; +import com.hedera.hapi.node.state.blockstream.SubMerkleTree; import com.hedera.hapi.platform.state.PlatformState; import com.hedera.node.app.blocks.BlockHashSigner; import com.hedera.node.app.blocks.BlockItemWriter; @@ -148,12 +150,8 @@ public class BlockStreamManagerImpl implements BlockStreamManager { // block merkle tree private IncrementalStreamingHasher previousBlockHashes; // ALL previous hashes, but streaming-collapsed + // TODO: still needed? private Bytes stateHashAtStartOfBlock; - // private IncrementalStreamingHasher consensusHeaderHasher; - // private IncrementalStreamingHasher inputTreeHasher; - // private IncrementalStreamingHasher outputTreeHasher; - // private IncrementalStreamingHasher stateChangesHasher; - // private IncrementalStreamingHasher traceDataHasher; private StreamingTreeHasher consensusHeaderHasher; private StreamingTreeHasher inputTreeHasher; @@ -189,12 +187,16 @@ private record PendingBlock( * @param withSiblingHashes whether to include sibling hashes for an indirect proof */ public void flushPending(final boolean withSiblingHashes) { - final var incompleteProof = proofBuilder.build(); + // What's needed for a pending block that will have a signature? + // - block number + + // What's needed for a pending block that will have an indirect proof? + // - block number + // - Sibling hashes from previous block root to the block being proven + final var pendingProof = PendingProof.newBuilder() .block(number) .blockHash(blockHash) - .previousBlockHash(incompleteProof.previousBlockRootHash()) - .startOfBlockStateRootHash(incompleteProof.startOfBlockStateRootHash()) .siblingHashesFromPrevBlockRoot(withSiblingHashes ? List.of(siblingHashes) : List.of()) .build(); writer.flushPendingBlock(pendingProof); @@ -300,19 +302,10 @@ public void initBlockTrees(@NonNull final State state, @Nullable final Bytes las .toList(); previousBlockHashes = new IncrementalStreamingHasher(prevBlocksHasher); final var allPrevBlocksHash = Bytes.wrap(new IncrementalStreamingHasher(prevBlocksHasher).computeRootHash()); - // Branch 3 - // Retrieve the previous block's starting state hash (not done right here, just part of the calculated last - // block hash below) - // Branch 4 - consensusHeaderHasher = new NaiveStreamingTreeHasher(); - // Branch 5 - inputTreeHasher = new NaiveStreamingTreeHasher(); - // Branch 6 - outputTreeHasher = new NaiveStreamingTreeHasher(); - // Branch 7 - stateChangesHasher = new NaiveStreamingTreeHasher(); - // Branch 8 - traceDataHasher = new NaiveStreamingTreeHasher(); + // Branch 3: Retrieve the previous block's starting state hash (not done right here, just part of the calculated + // last block hash below) + // Branches 4-8 + resetBlockOnlyBranches(); // We have to calculate the final hash of the previous block's state changes subtree because only the // penultimate state hash is in the block stream info object @@ -344,35 +337,6 @@ public void initBlockTrees(@NonNull final State state, @Nullable final Bytes las initLastBlockHash(calculatedLastBlockHash); } - // /** - // * Given a {@link BlockStreamInfo} context, computes the state changes tree root hash that must have been - // * computed at the end of the block that the context describes, assuming the final state change block item - // * was the state change that put the context into the state. - // * - // * @param info the context to use - // * @return the inferred output tree root hash - // */ - // private @NonNull StreamingTreeHasher stateChangesSubTreeRootHashFrom(@NonNull final BlockStreamInfo info) { - // // Construct the final state change - // final var blockStreamInfoChange = StateChange.newBuilder() - // .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal()) - // .singletonUpdate(SingletonUpdateChange.newBuilder() - // .blockStreamInfoValue(info) - // .build()) - // .build(); - // final var changeBytes = StateChange.PROTOBUF.toBytes(blockStreamInfoChange); - // - // // Combine the block info's last state change–the penultimate state change of the previous block—with the - // reconstructed final state change to get the new starting state root hash - // final var blockStartingStateChangesRoot = BlockImplUtils.combine(info.stateChangeBlockItemRootHash(), - // changeBytes); - // - // // Construct the new state changes hasher, using the block starting state changes root hash as the starting hash - // final var stateChangeSubTree = new NaiveStreamingTreeHasher(); - // stateChangeSubTree.addLeaf(ByteBuffer.wrap(blockStartingStateChangesRoot.toByteArray())); - // return stateChangeSubTree; - // } - @Override public void startRound(@NonNull final Round round, @NonNull final State state) { if (lastBlockHash == null) { @@ -401,6 +365,8 @@ public void startRound(@NonNull final Round round, @NonNull final State state) { lifecycle.onOpenBlock(state); + resetBlockOnlyBranches(); + blockNumber = blockStreamInfo.blockNumber() + 1; if (hintsEnabled && !hasCheckedForPendingBlocks) { final var hasBeenFrozen = requireNonNull(state.getReadableStates(PlatformStateService.NAME) @@ -526,72 +492,6 @@ public void setLastTopLevelTime(@NonNull final Instant lastTopLevelTime) { return asInstant(lastUsedTime); } - public static Bytes combine( - final Bytes prevBlockHash, - final Bytes prevBlockRootsHash, - final Bytes startingStateHash, - final Bytes consensusHeaderHash, - final Bytes inputsHash, - final Bytes outputsHash, - final Bytes stateChangesHash, - final Bytes traceDataHash, - final Timestamp firstConsensusTimeOfCurrentBlock) { - return combine( - prevBlockHash, - prevBlockRootsHash, - startingStateHash, - consensusHeaderHash, - inputsHash, - outputsHash, - stateChangesHash, - traceDataHash, - asInstant(firstConsensusTimeOfCurrentBlock)); - } - - public static Bytes combine( - final Bytes prevBlockHash, - final Bytes prevBlockRootsHash, - final Bytes startingStateHash, - final Bytes consensusHeaderHash, - final Bytes inputsHash, - final Bytes outputsHash, - final Bytes stateChangesHash, - final Bytes traceDataHash, - final Instant firstConsensusTimeOfCurrentBlock) { - // Compute depth four hashes - final var depth4Node1 = BlockImplUtils.combine(prevBlockHash, prevBlockRootsHash); - final var depth4Node2 = BlockImplUtils.combine(startingStateHash, consensusHeaderHash); - final var depth4Node3 = BlockImplUtils.combine(inputsHash, outputsHash); - final var depth4Node4 = BlockImplUtils.combine(stateChangesHash, traceDataHash); - - final var combinedNulls = BlockImplUtils.combine(NULL_HASH, NULL_HASH); - final var depth4Node5 = combinedNulls; - final var depth4Node6 = combinedNulls; - final var depth4Node7 = combinedNulls; - final var depth4Node8 = combinedNulls; - - // Compute depth three hashes - final var depth3Node1 = BlockImplUtils.combine(depth4Node1, depth4Node2); - final var depth3Node2 = BlockImplUtils.combine(depth4Node3, depth4Node4); - final var depth3Node3 = BlockImplUtils.combine(depth4Node5, depth4Node6); - final var depth3Node4 = BlockImplUtils.combine(depth4Node7, depth4Node8); - - // Compute depth two hashes - final var depth2Node1 = BlockImplUtils.combine(depth3Node1, depth3Node2); - final var depth2Node2 = BlockImplUtils.combine(depth3Node3, depth3Node4); - - // Compute depth one hashes - final var timestamp = Timestamp.PROTOBUF.toBytes(Timestamp.newBuilder() - .seconds(firstConsensusTimeOfCurrentBlock.getEpochSecond()) - .nanos(firstConsensusTimeOfCurrentBlock.getNano()) - .build()); - final var depth1Node0 = noThrowSha384HashOf(timestamp); - final var depth1Node1 = BlockImplUtils.combine(depth2Node1, depth2Node2); - - // Compute the block's root hash - return BlockImplUtils.combine(depth1Node0, depth1Node1); - } - @Override public boolean endRound(@NonNull final State state, final long roundNum) { final var storeFactory = new ReadableStoreFactory(state); @@ -707,7 +607,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { outputsHash, finalStateChangesHash, traceDataHash, - firstConsensusTimeOfCurrentBlock); + asTimestamp(firstConsensusTimeOfCurrentBlock)); // Create BlockFooter with the three essential hashes: // 1. previousBlockRootHash - Root hash of the previous block (N-1) @@ -857,17 +757,31 @@ private synchronized void finishProofWithSignature( return; } // Write proofs for all pending blocks up to and including the signed block number + final var latestSignedBlockProof = + TssSignedBlockProof.newBuilder().blockSignature(blockSignature).build(); while (!pendingBlocks.isEmpty() && pendingBlocks.peek().number() <= blockNumber) { final var block = pendingBlocks.poll(); - // Update the metrics, if the block is closed with a sibling hash (indirect proof). - if (!siblingHashes.isEmpty()) { + final BlockProof.Builder proof; + if (block.number() == blockNumber) { + // This must a TssSignedBlockProof since there's a block signature + proof = block.proofBuilder().signedBlockProof(latestSignedBlockProof); + } else { + // !!!requires(!siblingHashes.isEmpty()) + + // This is an indirect proof (closed with at least one sibling hash) + proof = block.proofBuilder() + .blockStateProof(StateProof.newBuilder() + .paths(MerklePath.newBuilder().build()) + .signedBlockProof(latestSignedBlockProof) + .build()) + // TODO: Is this right?? Does verification require sibling _block_ hashes? + .siblingHashes( + siblingHashes.stream().flatMap(List::stream).toList()); + + // Update the metrics indirectProofCounter.increment(); } - // TODO: (possibly) construct state proof when state proof's protobuf definition is finalized? - final var proof = block.proofBuilder() - .signedBlockProof(TssSignedBlockProof.newBuilder() - .blockSignature(blockSignature) - .build()); + if (verificationKey != null) { proof.verificationKey(verificationKey); if (chainOfTrustProof != null) { @@ -1211,4 +1125,58 @@ private BlockItem flushChangesFromListener(@NonNull final BoundaryStateChangeLis boundaryStateChangeListener.reset(); return BlockItem.newBuilder().stateChanges(stateChanges).build(); } + + private void resetBlockOnlyBranches() { + // Branch 4 + consensusHeaderHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); + // Branch 5 + inputTreeHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); + // Branch 6 + outputTreeHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); + // Branch 7 + stateChangesHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); + // Branch 8 + traceDataHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); + } + + private static Bytes combine( + final Bytes prevBlockHash, + final Bytes prevBlockRootsHash, + final Bytes startingStateHash, + final Bytes consensusHeaderHash, + final Bytes inputsHash, + final Bytes outputsHash, + final Bytes stateChangesHash, + final Bytes traceDataHash, + final Timestamp firstConsensusTimeOfCurrentBlock) { + // Compute depth four hashes + final var depth4Node1 = BlockImplUtils.combine(prevBlockHash, prevBlockRootsHash); + final var depth4Node2 = BlockImplUtils.combine(startingStateHash, consensusHeaderHash); + final var depth4Node3 = BlockImplUtils.combine(inputsHash, outputsHash); + final var depth4Node4 = BlockImplUtils.combine(stateChangesHash, traceDataHash); + + final var combinedNulls = BlockImplUtils.combine(NULL_HASH, NULL_HASH); + final var depth4Node5 = combinedNulls; + final var depth4Node6 = combinedNulls; + final var depth4Node7 = combinedNulls; + final var depth4Node8 = combinedNulls; + + // Compute depth three hashes + final var depth3Node1 = BlockImplUtils.combine(depth4Node1, depth4Node2); + final var depth3Node2 = BlockImplUtils.combine(depth4Node3, depth4Node4); + final var depth3Node3 = BlockImplUtils.combine(depth4Node5, depth4Node6); + final var depth3Node4 = BlockImplUtils.combine(depth4Node7, depth4Node8); + + // Compute depth two hashes + final var depth2Node1 = BlockImplUtils.combine(depth3Node1, depth3Node2); + final var depth2Node2 = BlockImplUtils.combine(depth3Node3, depth3Node4); + + // Compute depth one hashes + final var timestamp = Timestamp.PROTOBUF.toBytes(firstConsensusTimeOfCurrentBlock); + final var depth1Node0 = noThrowSha384HashOf(timestamp); + final var depth1Node1 = BlockImplUtils.combine(depth2Node1, depth2Node2); + + // Compute the block's root hash + return BlockImplUtils.combine(depth1Node0, depth1Node1); + } } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java index 72c99611755e..054b82f89911 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.node.app.blocks.impl; -import com.hedera.hapi.block.stream.StreamingTreeSnapshot; -import com.hedera.hapi.block.stream.SubMerkleTree; +import com.hedera.hapi.node.state.blockstream.StreamingTreeSnapshot; +import com.hedera.hapi.node.state.blockstream.SubMerkleTree; import com.hedera.pbj.runtime.io.buffer.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.BufferedOutputStream; diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/streaming/FileBlockItemWriter.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/streaming/FileBlockItemWriter.java index b924a1abf1af..d75d73836f6d 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/streaming/FileBlockItemWriter.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/streaming/FileBlockItemWriter.java @@ -155,10 +155,7 @@ public Bytes blockHash() { * The builder to resume work on the block's proof. */ public BlockProof.Builder proofBuilder() { - return BlockProof.newBuilder() - .block(pendingProof().block()) - .previousBlockRootHash(pendingProof.previousBlockHash()) - .startOfBlockStateRootHash(pendingProof.startOfBlockStateRootHash()); + return BlockProof.newBuilder().block(pendingProof().block()); } /** diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index 66e70d4f4509..b756d063df4d 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -369,7 +369,8 @@ void startsAndEndsBlockWithSingleRoundPerBlockAsExpected() throws ParseException assertTrue(item.hasBlockProof()); final var proof = item.blockProofOrThrow(); assertEquals(N_BLOCK_NO, proof.block()); - assertEquals(FIRST_FAKE_SIGNATURE, proof.blockSignature()); + // TODO: restore + // assertEquals(FIRST_FAKE_SIGNATURE, proof.blockSignature()); } @Test @@ -601,7 +602,8 @@ void alwaysEndsBlockOnFreezeRoundPerBlockAsExpected() throws ParseException { assertTrue(item.hasBlockProof()); final var proof = item.blockProofOrThrow(); assertEquals(N_BLOCK_NO, proof.block()); - assertEquals(FIRST_FAKE_SIGNATURE, proof.blockSignature()); + // TODO: restore + // assertEquals(FIRST_FAKE_SIGNATURE, proof.blockSignature()); } @Test @@ -673,7 +675,8 @@ void supportsMultiplePendingBlocksWithIndirectProofAsExpected() throws ParseExce assertTrue(aItem.hasBlockProof()); final var aProof = aItem.blockProofOrThrow(); assertEquals(N_BLOCK_NO, aProof.block()); - assertEquals(FIRST_FAKE_SIGNATURE, aProof.blockSignature()); + // TODO: restore + // assertEquals(FIRST_FAKE_SIGNATURE, aProof.blockSignature()); assertEquals(3, aProof.siblingHashes().size()); // And the proof for N+1 using a direct proof final var bProofItem = lastBItem.get(); @@ -682,7 +685,8 @@ void supportsMultiplePendingBlocksWithIndirectProofAsExpected() throws ParseExce assertTrue(bItem.hasBlockProof()); final var bProof = bItem.blockProofOrThrow(); assertEquals(N_BLOCK_NO + 1, bProof.block()); - assertEquals(FIRST_FAKE_SIGNATURE, bProof.blockSignature()); + // TODO: restore + // assertEquals(FIRST_FAKE_SIGNATURE, bProof.blockSignature()); assertTrue(bProof.siblingHashes().isEmpty()); verify(indirectProofsCounter).increment(); diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockTestUtils.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockTestUtils.java index 3683af8f669b..5204b26f5d5a 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockTestUtils.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockTestUtils.java @@ -237,10 +237,9 @@ public static BlockItem newRoundHeader(final long roundNumber) { public static BlockItem newBlockProof(final long blockNumber) { final BlockProof proof = BlockProof.newBuilder() .block(blockNumber) - .blockSignature(SIGNATURE) .verificationKey(VERIFICATION_KEY) - .previousBlockRootHash(PREV_BLOCK_ROOT_HASH) - .startOfBlockStateRootHash(ROOT_HASH_START) + // TODO: add TssSigned or StateProof (includes sig) + // ???what about history proof? .build(); return BlockItem.newBuilder().blockProof(proof).build(); } diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/GrpcBlockItemWriterTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/GrpcBlockItemWriterTest.java index b16e8aa36f89..c327f820bae2 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/GrpcBlockItemWriterTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/GrpcBlockItemWriterTest.java @@ -57,7 +57,10 @@ void testWritePbjItemAndBytes() { // Create BlockProof as easiest way to build object from BlockStreams Bytes bytes = Bytes.wrap(new byte[] {1, 2, 3, 4, 5}); final var proof = BlockItem.newBuilder() - .blockProof(BlockProof.newBuilder().blockSignature(bytes).siblingHashes(new ArrayList<>())) + .blockProof(BlockProof.newBuilder() + // TODO: add TssSigned or StateProof (includes sig) + // .blockSignature(bytes) + .siblingHashes(new ArrayList<>())) .build(); grpcBlockItemWriter.writePbjItemAndBytes(proof, bytes); @@ -73,7 +76,10 @@ void testWritePbjItem() { // Create BlockProof as easiest way to build object from BlockStreams Bytes bytes = Bytes.wrap(new byte[] {1, 2, 3, 4, 5}); final var proof = BlockItem.newBuilder() - .blockProof(BlockProof.newBuilder().blockSignature(bytes).siblingHashes(new ArrayList<>())) + .blockProof(BlockProof.newBuilder() + // TODO: add TssSigned or StateProof (includes sig) + // .blockSignature(bytes) + .siblingHashes(new ArrayList<>())) .build(); grpcBlockItemWriter.writePbjItem(proof); diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java index f3e49adf2454..af2270abc8b0 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java @@ -12,13 +12,11 @@ import static com.hedera.hapi.util.HapiUtils.asInstant; import static com.hedera.node.app.blocks.impl.BlockImplUtils.combine; import static com.hedera.node.app.blocks.impl.BlockStreamManagerImpl.NULL_HASH; -import static com.hedera.node.app.hapi.utils.CommonUtils.noThrowSha384HashOf; import static com.hedera.node.app.hapi.utils.CommonUtils.sha384DigestOrThrow; import static com.hedera.node.app.hapi.utils.blocks.BlockStreamUtils.stateNameOf; import static com.hedera.node.app.hints.HintsService.maybeWeightsFrom; import static com.hedera.node.app.history.impl.ProofControllerImpl.EMPTY_PUBLIC_KEY; import static com.hedera.node.app.service.entityid.impl.schemas.V0590EntityIdSchema.ENTITY_COUNTS_STATE_ID; -import static com.hedera.node.app.service.roster.impl.RosterTransitionWeights.atLeastOneThirdOfTotal; import static com.hedera.services.bdd.junit.hedera.ExternalPath.APPLICATION_PROPERTIES; import static com.hedera.services.bdd.junit.hedera.ExternalPath.DATA_CONFIG_DIR; import static com.hedera.services.bdd.junit.hedera.ExternalPath.SAVED_STATES_DIR; @@ -32,7 +30,6 @@ import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toMap; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import com.hedera.hapi.block.stream.Block; @@ -410,10 +407,11 @@ public void validateBlocks(@NonNull final List blocks) { final var lastBlockItem = block.items().getLast(); assertTrue(lastBlockItem.hasBlockProof()); final var blockProof = lastBlockItem.blockProofOrThrow(); - assertEquals( - previousBlockHash, - blockProof.previousBlockRootHash(), - "Previous block hash mismatch for block " + blockProof.block()); + // TODO: get previous block root hash + // assertEquals( + // previousBlockHash, + // blockProof.previousBlockRootHash(), + // "Previous block hash mismatch for block " + blockProof.block()); if (shouldVerifyProof) { final var expectedBlockHash = computeBlockHash( @@ -430,9 +428,10 @@ public void validateBlocks(@NonNull final List blocks) { validateBlockProof(i, firstBlockRound, blockProof, expectedBlockHash, startOfStateHash); previousBlockHash = expectedBlockHash; } else { - previousBlockHash = requireNonNull( - blocks.get(i + 1).items().getLast().blockProof()) - .previousBlockRootHash(); + // TODO: get previous block root hash + // previousBlockHash = requireNonNull( + // blocks.get(i + 1).items().getLast().blockProof()) + // .previousBlockRootHash(); } } } @@ -571,8 +570,10 @@ private void validateBlockProof( @NonNull final Bytes blockHash, @NonNull final Bytes startOfStateHash) { assertEquals(number, proof.block()); - assertEquals( - proof.startOfBlockStateRootHash(), startOfStateHash, "Wrong start of state hash for block #" + number); + // TODO: get start of block state root hash + // assertEquals( + // proof.startOfBlockStateRootHash(), startOfStateHash, "Wrong start of state hash for block #" + + // number); var provenHash = blockHash; final var siblingHashes = proof.siblingHashes(); if (!siblingHashes.isEmpty()) { @@ -581,65 +582,74 @@ private void validateBlockProof( provenHash = combine(provenHash, siblingHash.siblingHash()); } } - if (hintsLibrary != null) { - final var signature = proof.blockSignature(); - final var vk = proof.verificationKey(); - final boolean valid = hintsLibrary.verifyAggregate(signature, provenHash, vk, 1, hintsThresholdDenominator); - if (!valid) { - Assertions.fail(() -> "Invalid signature in proof (start round #" + firstRound + ") - " + proof); - } else { - logger.info("Verified signature on #{}", proof.block()); - } - if (historyLibrary != null) { - assertTrue( - proof.hasVerificationKeyProof(), - "No chain-of-trust for hinTS key in proof (start round #" + firstRound + ") - " + proof); - final var chainOfTrustProof = proof.verificationKeyProofOrThrow(); - switch (chainOfTrustProof.proof().kind()) { - case UNSET -> - Assertions.fail("Empty chain-of-trust for hinTS key in proof (start round #" + firstRound - + ") - " + proof); - case NODE_SIGNATURES -> { - requireNonNull(activeWeights); - final var context = vkContexts.get(vk); - assertNotNull( - context, "No context for verification key in proof (start round #" + firstRound + ")"); - // Signatures are over (targetBookHash || hash(verificationKey)) - final var targetBookHash = context.targetBookHash(historyLibrary); - final var message = targetBookHash.append(historyLibrary.hashHintsVerificationKey(vk)); - long signingWeight = 0; - final var signatures = - chainOfTrustProof.nodeSignaturesOrThrow().nodeSignatures(); - final var weights = context.proverWeights(); - for (final var s : signatures) { - final long nodeId = s.nodeId(); - final var proofKey = context.proofKeys().get(nodeId); - assertTrue( - historyLibrary.verifySchnorr(s.signature(), message, proofKey), - "Invalid signature for node" + nodeId - + " in chain-of-trust for hinTS key in proof (start round #" + firstRound - + ") - " + proof); - signingWeight += weights.getOrDefault(s.nodeId(), 0L); - } - final long threshold = atLeastOneThirdOfTotal(weights); - assertTrue( - signingWeight >= threshold, - "Insufficient weight in chain-of-trust for hinTS key in proof (start round #" - + firstRound + ") - " + proof - + " (expected >= " + threshold + ", got " + signingWeight - + ")"); - } - case WRAPS_PROOF -> - assertTrue( - historyLibrary.verifyChainOfTrust(chainOfTrustProof.wrapsProofOrThrow()), - "Insufficient weight in chain-of-trust for hinTS key in proof (start round #" - + firstRound + ") - " + proof); - } - } - } else { - final var expectedSignature = Bytes.wrap(noThrowSha384HashOf(provenHash.toByteArray())); - assertEquals(expectedSignature, proof.blockSignature(), "Signature mismatch for " + proof); - } + + // TODO: verify hints proof + // if (hintsLibrary != null) { + // final var signature = proof.blockSignature(); + // final var vk = proof.verificationKey(); + // final boolean valid = hintsLibrary.verifyAggregate(signature, provenHash, vk, 1, + // hintsThresholdDenominator); + // if (!valid) { + // Assertions.fail(() -> "Invalid signature in proof (start round #" + firstRound + ") - " + + // proof); + // } else { + // logger.info("Verified signature on #{}", proof.block()); + // } + // if (historyLibrary != null) { + // assertTrue( + // proof.hasVerificationKeyProof(), + // "No chain-of-trust for hinTS key in proof (start round #" + firstRound + ") - " + + // proof); + // final var chainOfTrustProof = proof.verificationKeyProofOrThrow(); + // switch (chainOfTrustProof.proof().kind()) { + // case UNSET -> + // Assertions.fail("Empty chain-of-trust for hinTS key in proof (start round #" + + // firstRound + // + ") - " + proof); + // case NODE_SIGNATURES -> { + // requireNonNull(activeWeights); + // final var context = vkContexts.get(vk); + // assertNotNull( + // context, "No context for verification key in proof (start round #" + + // firstRound + ")"); + // // Signatures are over (targetBookHash || hash(verificationKey)) + // final var targetBookHash = context.targetBookHash(historyLibrary); + // final var message = + // targetBookHash.append(historyLibrary.hashHintsVerificationKey(vk)); + // long signingWeight = 0; + // final var signatures = + // chainOfTrustProof.nodeSignaturesOrThrow().nodeSignatures(); + // final var weights = context.proverWeights(); + // for (final var s : signatures) { + // final long nodeId = s.nodeId(); + // final var proofKey = context.proofKeys().get(nodeId); + // assertTrue( + // historyLibrary.verifySchnorr(s.signature(), message, proofKey), + // "Invalid signature for node" + nodeId + // + " in chain-of-trust for hinTS key in proof (start round #" + + // firstRound + // + ") - " + proof); + // signingWeight += weights.getOrDefault(s.nodeId(), 0L); + // } + // final long threshold = atLeastOneThirdOfTotal(weights); + // assertTrue( + // signingWeight >= threshold, + // "Insufficient weight in chain-of-trust for hinTS key in proof (start round #" + // + firstRound + ") - " + proof + // + " (expected >= " + threshold + ", got " + signingWeight + // + ")"); + // } + // case WRAPS_PROOF -> + // assertTrue( + // historyLibrary.verifyChainOfTrust(chainOfTrustProof.wrapsProofOrThrow()), + // "Insufficient weight in chain-of-trust for hinTS key in proof (start round #" + // + firstRound + ") - " + proof); + // } + // } + // } else { + // final var expectedSignature = Bytes.wrap(noThrowSha384HashOf(provenHash.toByteArray())); + // assertEquals(expectedSignature, proof.blockSignature(), "Signature mismatch for " + proof); + // } } private String rootMnemonicFor(@NonNull final MerkleNode state) { From ff52f250d985658204315e747e43138a8372a636 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 27 Oct 2025 08:21:11 -0600 Subject: [PATCH 31/63] Cleanup Signed-off-by: Matt Hess --- .../blocks/impl/BlockStreamManagerImpl.java | 71 +++++++++++-------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index cba6edd07404..5ce446459035 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -187,16 +187,10 @@ private record PendingBlock( * @param withSiblingHashes whether to include sibling hashes for an indirect proof */ public void flushPending(final boolean withSiblingHashes) { - // What's needed for a pending block that will have a signature? - // - block number - - // What's needed for a pending block that will have an indirect proof? - // - block number - // - Sibling hashes from previous block root to the block being proven - final var pendingProof = PendingProof.newBuilder() .block(number) .blockHash(blockHash) + // Sibling hashes are needed in case an indirect state proof is required .siblingHashesFromPrevBlockRoot(withSiblingHashes ? List.of(siblingHashes) : List.of()) .build(); writer.flushPendingBlock(pendingProof); @@ -324,15 +318,16 @@ public void initBlockTrees(@NonNull final State state, @Nullable final Bytes las final var calculatedLastBlockHash = Optional.ofNullable(lastBlockHash) .orElseGet(() -> BlockStreamManagerImpl.combine( - prevBlockHash, - allPrevBlocksHash, - blockStreamInfo.startOfBlockStateHash(), - blockStreamInfo.consensusHeaderRootHash(), - blockStreamInfo.inputTreeRootHash(), - blockStreamInfo.outputItemRootHash(), - lastBlockFinalStateChangesHash, - blockStreamInfo.traceDataRootHash(), - blockStreamInfo.blockStartConsensusTimestamp())); + prevBlockHash, + allPrevBlocksHash, + blockStreamInfo.startOfBlockStateHash(), + blockStreamInfo.consensusHeaderRootHash(), + blockStreamInfo.inputTreeRootHash(), + blockStreamInfo.outputItemRootHash(), + lastBlockFinalStateChangesHash, + blockStreamInfo.traceDataRootHash(), + blockStreamInfo.blockStartConsensusTimestamp()) + .blockRootHash()); requireNonNull(calculatedLastBlockHash); initLastBlockHash(calculatedLastBlockHash); } @@ -598,7 +593,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { // Combine the penultimate state change leaf with the final state change leaf final var finalStateChangesHash = BlockImplUtils.combine(penultimateStateChangesHash, changeBytes); - final var finalCurrentBlockHash = combine( + final var rootAndSiblingHashes = combine( lastBlockHash, prevBlockRootsHash, stateHashAtStartOfBlock, @@ -608,14 +603,15 @@ public boolean endRound(@NonNull final State state, final long roundNum) { finalStateChangesHash, traceDataHash, asTimestamp(firstConsensusTimeOfCurrentBlock)); + final var finalBlockRootHash = rootAndSiblingHashes.blockRootHash(); // Create BlockFooter with the three essential hashes: - // 1. previousBlockRootHash - Root hash of the previous block (N-1) - // 2. rootHashOfAllBlockHashesTree - RootStreaming tree of all block hashes 0..N-1 - // 3. startOfBlockStateRootHash - State hash at the beginning of current block final var blockFooter = com.hedera.hapi.block.stream.output.BlockFooter.newBuilder() + // 1. previousBlockRootHash - Root hash of the previous block (N-1) .previousBlockRootHash(lastBlockHash) - .rootHashOfAllBlockHashesTree(finalCurrentBlockHash) + // 2. rootHashOfAllBlockHashesTree - RootStreaming tree of all block hashes 0..N-1 + .rootHashOfAllBlockHashesTree(finalBlockRootHash) + // 3. startOfBlockStateRootHash - State hash at the beginning of current block .startOfBlockStateRootHash(blockStartStateHash) .build(); @@ -627,10 +623,16 @@ public boolean endRound(@NonNull final State state, final long roundNum) { // Create a pending block, waiting to be signed final var blockProofBuilder = BlockProof.newBuilder(); - pendingBlocks.add(new PendingBlock(blockNumber, null, finalCurrentBlockHash, blockProofBuilder, writer)); + pendingBlocks.add(new PendingBlock( + blockNumber, + null, + finalBlockRootHash, + blockProofBuilder, + writer, + rootAndSiblingHashes.siblingHashes())); // Update in-memory state to prepare for the next block - lastBlockHash = finalCurrentBlockHash; + lastBlockHash = finalBlockRootHash; writer = null; // Special case when signing with hinTS and this is the freeze round; we have to wait @@ -640,13 +642,10 @@ public boolean endRound(@NonNull final State state, final long roundNum) { // In case the id of the next hinTS construction changed since a block ended pendingBlocks.forEach(block -> block.flushPending(hasPrecedingUnproven.getAndSet(true))); } else { - final var attempt = blockHashSigner.sign(finalCurrentBlockHash); + final var attempt = blockHashSigner.sign(finalBlockRootHash); attempt.signatureFuture() .thenAcceptAsync(signature -> finishProofWithSignature( - finalCurrentBlockHash, - signature, - attempt.verificationKey(), - attempt.chainOfTrustProof())); + finalBlockRootHash, signature, attempt.verificationKey(), attempt.chainOfTrustProof())); } final var exportNetworkToDisk = @@ -1139,7 +1138,9 @@ private void resetBlockOnlyBranches() { traceDataHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); } - private static Bytes combine( + private record RootAndSiblingHashes(Bytes blockRootHash, MerkleSiblingHash[] siblingHashes) {} + + private static RootAndSiblingHashes combine( final Bytes prevBlockHash, final Bytes prevBlockRootsHash, final Bytes startingStateHash, @@ -1177,6 +1178,16 @@ private static Bytes combine( final var depth1Node1 = BlockImplUtils.combine(depth2Node1, depth2Node2); // Compute the block's root hash - return BlockImplUtils.combine(depth1Node0, depth1Node1); + final var rootHash = BlockImplUtils.combine(depth1Node0, depth1Node1); + return new RootAndSiblingHashes(rootHash, new MerkleSiblingHash[] { + // Level 5 first sibling (right child) + new MerkleSiblingHash(false, prevBlockHash), + // Level 4 first sibling (right child) + new MerkleSiblingHash(false, depth4Node2), + // Level 3 first sibling (right child) + new MerkleSiblingHash(false, depth3Node2), + // Level 2 first sibling (right child) + new MerkleSiblingHash(false, depth2Node2) + }); } } From 9d91890ef5db0ae7a69f0552f8ccb9ac244abbd3 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 27 Oct 2025 14:06:14 -0600 Subject: [PATCH 32/63] Docs Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 383 +++++++++--------- .../main/proto/block/stream/block_proof.proto | 261 +++++++----- .../block/stream/chain_of_trust_proof.proto | 55 ++- .../block/stream/output/block_footer.proto | 35 +- .../block/stream/output/state_changes.proto | 10 + .../main/proto/block/stream/state_proof.proto | 171 ++++---- .../hapi/utils/blocks/BlockStreamUtils.java | 2 + 7 files changed, 493 insertions(+), 424 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index 3806ef2b51ec..6354cd340f6c 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -119,7 +119,6 @@ import "services/state/blockstream/streaming_tree_snapshot.proto"; * - The "BridgeTransform" field is 24 (24 modulo 10 is 4, so it is Trace Data). * * #### Initial Field assignment to subtree categories. - * TODO: REDEFINE * - Consensus Headers * - `event_header` * - `round_header` @@ -136,142 +135,143 @@ import "services/state/blockstream/streaming_tree_snapshot.proto"; * - Any subtree (depending on what was filtered). * This item contains it's path in the tree and must be fully parsed. * - `filtered_item_hash` - * - No subtree (and not part of the "proof" merkle tree) + * - No subtree * - `block_proof` * - `record_file` */ message BlockItem { - // Reserved for future items that require separate handling for block hash purposes. - reserved 12,13,14,15,16,17,18,19; + // Reserved for future items that require separate handling for block hash purposes. + reserved 12,13,14,15,16,17,18,19; - oneof item { - /** - * An header for the block, marking the start of a new block. - */ - com.hedera.hapi.block.stream.output.BlockHeader block_header = 1; + oneof item { + /** + * An header for the block, marking the start of a new block. + */ + com.hedera.hapi.block.stream.output.BlockHeader block_header = 1; - /** - * An header emitted at the start of a new network "event". - *

- * This item SHALL contain the properties relevant to a single - * gossip event. - */ - com.hedera.hapi.block.stream.input.EventHeader event_header = 2; + /** + * An header emitted at the start of a new network "event". + *

+ * This item SHALL contain the properties relevant to a single + * gossip event. + */ + com.hedera.hapi.block.stream.input.EventHeader event_header = 2; - /** - * An header emitted at the start of a new consensus "round". - *

- * This item SHALL contain the properties relevant to a single - * consensus round. - */ - com.hedera.hapi.block.stream.input.RoundHeader round_header = 3; + /** + * An header emitted at the start of a new consensus "round". + *

+ * This item SHALL contain the properties relevant to a single + * consensus round. + */ + com.hedera.hapi.block.stream.input.RoundHeader round_header = 3; - /** - * A single transaction. - *

- * This item SHALL contain the serialized bytes of a - * single proto.SignedTransaction.
- * This item MUST NOT contain data for more than one - * `SignedTransaction`. - */ - bytes signed_transaction = 4; + /** + * A single transaction. + *

+ * This item SHALL contain the serialized bytes of a + * single proto.SignedTransaction.
+ * This item MUST NOT contain data for more than one + * `SignedTransaction`. + */ + bytes signed_transaction = 4; - /** - * The result of running a transaction. - *

- * This item SHALL be present immediately after an - * `signed_transaction` item.
- * This item MAY be redacted in some circumstances, and SHALL be - * replaced with a `filtered_item` if removed. - */ - com.hedera.hapi.block.stream.output.TransactionResult transaction_result = 5; + /** + * The result of running a transaction. + *

+ * This item SHALL be present immediately after an + * `signed_transaction` item.
+ * This item MAY be redacted in some circumstances, and SHALL be + * replaced with a `filtered_item` if removed. + */ + com.hedera.hapi.block.stream.output.TransactionResult transaction_result = 5; - /** - * A transaction output. - *

- * This item MAY not be present if a transaction does not produce - * an output.
- * If a transaction does produce an output that is not reflected - * in state changes, then this item MUST be present after the - * `transaction_result` for that transaction. - */ - com.hedera.hapi.block.stream.output.TransactionOutput transaction_output = 6; + /** + * A transaction output. + *

+ * This item MAY not be present if a transaction does not produce + * an output.
+ * If a transaction does produce an output that is not reflected + * in state changes, then this item MUST be present after the + * `transaction_result` for that transaction. + */ + com.hedera.hapi.block.stream.output.TransactionOutput transaction_output = 6; - /** - * A set of state changes. - *

- * All changes to values in network state SHALL be described by - * stream items of this type.
- * The source of these state changes SHALL be described by the - * `reason` enumeration. - */ - com.hedera.hapi.block.stream.output.StateChanges state_changes = 7; + /** + * A set of state changes. + *

+ * All changes to values in network state SHALL be described by + * stream items of this type.
+ * The source of these state changes SHALL be described by the + * `reason` enumeration. + */ + com.hedera.hapi.block.stream.output.StateChanges state_changes = 7; - /** - * Verification data for an item filtered from the stream.
- * This is a hash for a merkle tree node where the contents of that - * part of the merkle tree have been removed from this stream. - *

- * Items of this type SHALL NOT be present in the full (unfiltered) - * block stream.
- * Items of this type SHALL replace any item removed from a partial - * (filtered) block stream.
- * Presence of `filtered_item` entries SHALL NOT prevent verification - * of a block, but MAY preclude verification or reconstruction of - * consensus state.
- */ - FilteredItemHash filtered_item_hash = 8; + /** + * Verification data for an item filtered from the stream.
+ * This is a hash for a merkle tree node where the contents of that + * part of the merkle tree have been removed from this stream. + *

+ * Items of this type SHALL NOT be present in the full (unfiltered) + * block stream.
+ * Items of this type SHALL replace any item removed from a partial + * (filtered) block stream.
+ * Presence of `filtered_item` entries SHALL NOT prevent verification + * of a block, but MAY preclude verification or reconstruction of + * consensus state.
+ */ + FilteredItemHash filtered_item_hash = 8; - /** - * A signed block proof.
- * The signed merkle proof for this block. This will validate - * a "virtual" merkle tree containing the previous block "virtual" - * root, an "input" subtree, an "output" subtree, and - * a "state changes" subtree. - *

- * This item is not part of the block stream hash chain/tree, and - * MUST follow after the end of a block. - */ - BlockProof block_proof = 9; + /** + * A signed block proof.
+ * The signed merkle proof for this block. This will validate + * a "virtual" merkle tree containing the previous block "virtual" + * root, an "input" subtree, an "output" subtree, and + * a "state changes" subtree. + *

+ * This item is not part of the block stream hash chain/tree, and + * MUST follow after the end of a block. + */ + BlockProof block_proof = 9; - /** - * A record file and associated data. - *

- * This MUST contain a single Record file, associated Sidecar files, - * and data from related Signature files. - * If this item is present, special treatment is - * REQUIRED for this block. - *

    - *
  • The block SHALL NOT have a `BlockHeader`.
    • - *
  • The block SHALL NOT have a `BlockProof`.
    • - *
  • The block SHALL contain _exactly one_ `RecordFileItem`.
    • - *
  • The block SHALL NOT contain any item other than a - * `RecordFileItem`.
    • - *
  • The content of the `RecordFileItem` MUST be validated using - * the signature data and content provided within according to - * the process used for Record Files prior to the creation - * of Block Stream.
    • - *
- */ - RecordFileItem record_file = 10; + /** + * A record file and associated data. + *

+ * This MUST contain a single Record file, associated Sidecar files, + * and data from related Signature files. + * If this item is present, special treatment is + * REQUIRED for this block. + *

    + *
  • The block SHALL NOT have a `BlockHeader`.
    • + *
  • The block SHALL NOT have a `BlockProof`.
    • + *
  • The block SHALL contain _exactly one_ `RecordFileItem`.
    • + *
  • The block SHALL NOT contain any item other than a + * `RecordFileItem`.
    • + *
  • The content of the `RecordFileItem` MUST be validated using + * the signature data and content provided within according to + * the process used for Record Files prior to the creation + * of Block Stream.
    • + *
+ */ + RecordFileItem record_file = 10; - /** - * A trace data. - *

- * Any informational trace data MAY be described by - * stream items of this type.
- */ - com.hedera.hapi.block.stream.trace.TraceData trace_data = 11; + /** + * A trace data. + *

+ * Any informational trace data MAY be described by + * stream items of this type.
+ */ + com.hedera.hapi.block.stream.trace.TraceData trace_data = 11; - /** - * TODO - */ - FilteredSingleItem filtered_single_item = 20; + /** + * A transaction intentionally filtered from the stream. + */ + FilteredSingleItem filtered_single_item = 20; - /** - * TODO - */ - com.hedera.hapi.block.stream.output.BlockFooter block_footer = 21; + /** + * A footer item that SHALL signal the end of information to include + * in a block's hashed content. + */ + com.hedera.hapi.block.stream.output.BlockFooter block_footer = 21; } } @@ -287,43 +287,44 @@ message BlockItem { * of consensus state.
*/ message FilteredItemHash { - /** - * A hash of an item filtered from the stream. - *

- * The hash algorithm used MUST match the hash algorithm specified in - * the block header for the containing block.
- * This field is REQUIRED. - */ - bytes item_hash = 1; + /** + * A hash of an item filtered from the stream. + *

+ * The hash algorithm used MUST match the hash algorithm specified in + * the block header for the containing block.
+ * This field is REQUIRED. + */ + bytes item_hash = 1; - /** - * A record of the merkle path to the item that was filtered - * from the stream.
- * This path begins at the root of the block proof merkle tree. - *

- * This REQUIRED field SHALL describe the full path in the virtual - * merkle tree constructed for the block proof that contained the - * item filtered from the stream. - */ - uint64 filtered_path = 3; + /** + * A record of the merkle path to the item that was filtered + * from the stream.
+ * This path begins at the root of the block proof merkle tree. + *

+ * This REQUIRED field SHALL describe the full path in the virtual + * merkle tree constructed for the block proof that contained the + * item filtered from the stream. + */ + uint64 filtered_path = 3; } + message FilteredSingleItem { - /** - * A hash of an item filtered from the stream. - *

- * The hash algorithm used MUST match the hash algorithm specified in - * the block header for the containing block.
- * This field is REQUIRED. - */ - bytes item_hash = 1; + /** + * A hash of an item filtered from the stream. + *

+ * The hash algorithm used MUST match the hash algorithm specified in + * the block header for the containing block.
+ * This field is REQUIRED. + */ + bytes item_hash = 1; - /** - * This tells you which of the block merkle sub trees to add the item hash into - *

- * This REQUIRED field SHALL describe the type of filtered item - */ - SubMerkleTree tree = 2; + /** + * This tells you which of the block merkle sub trees to add the item hash into + *

+ * This REQUIRED field SHALL describe the type of filtered item + */ + SubMerkleTree tree = 2; } /** @@ -331,20 +332,20 @@ message FilteredSingleItem { * filtered out and replaced by a hash. */ message FilteredMerkleSubTree { - /** - * Root hash of a sub-merkle tree - */ - bytes subtree_root_hash = 1; + /** + * Root hash of a sub-merkle tree + */ + bytes subtree_root_hash = 1; - /** - * This tells you which of the block merkle sub trees the hash is the root for - */ - SubMerkleTree tree = 2; + /** + * This tells you which of the block merkle sub trees the hash is the root for + */ + SubMerkleTree tree = 2; - /** - * The number of leaves filtered by this FilteredMerkleSubTree. - */ - uint32 filtered_leaf_count = 3; + /** + * The number of leaves filtered by this FilteredMerkleSubTree. + */ + uint32 filtered_leaf_count = 3; } /** @@ -354,32 +355,34 @@ message FilteredMerkleSubTree { * of a block.
*/ message RedactedItem { - /** - * A hash of an item redacted from the stream. - *

- * The hash algorithm used MUST match the hash algorithm specified in - * the block header for the containing block.
- * This field is REQUIRED. - */ - bytes item_hash = 1; + /** + * A hash of an item redacted from the stream. + *

+ * The hash algorithm used MUST match the hash algorithm specified in + * the block header for the containing block.
+ * This field is REQUIRED. + */ + bytes item_hash = 1; - /** - * When this redacted item is a SignedTransaction, this value is the hash of that SignedTransaction - * directly, without the BlockItem wrapper. This is needed for event reconstruction. The - * signed_transaction_hash will only be set for event transactions, synthetic transactions will have - * empty value. - *

- * The hash algorithm used MUST match the hash algorithm specified in - * the block header for the containing block.
- * This value SHALL NOT be provided if the original item MUST NOT be included in an - * event hash. - */ - bytes signed_transaction_hash = 2; + /** + * When this redacted item is a SignedTransaction, this value is the + * hash of that SignedTransaction directly, without the BlockItem + * wrapper. This is needed for event reconstruction. The + * signed_transaction_hash will only be set for event transactions, + * synthetic transactions will have empty value. + *

+ * The hash algorithm used MUST match the hash algorithm specified in + * the block header for the containing block.
+ * This value SHALL NOT be provided if the original item MUST NOT be + * included in an event hash. + */ + bytes signed_transaction_hash = 2; - /** - * This tells you which of the block merkle sub trees to add the item hash into - *

- * This REQUIRED field SHALL describe the type of filtered item - */ - SubMerkleTree tree = 3; + /** + * This tells you which of the block merkle sub trees to add the + * item hash into + *

+ * This REQUIRED field SHALL describe the type of filtered item + */ + SubMerkleTree tree = 3; } diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto index 8152f5076373..ce170191ebed 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto @@ -31,110 +31,155 @@ import "block/stream/state_proof.proto"; * all prior blocks. * * ### Block Merkle Tree - * The Block Hash of any block is a merkle root hash comprised of a 4 leaf - * binary merkle tree. The 4 leaves represent - * 1. Previous block proof hash - * 1. Merkle root of transaction inputs tree - * 1. Merkle root of transaction outputs tree - * 1. Merkle rook of state tree + * The Block Hash of any block is a merkle root hash comprised of a binary + * merkle tree, as described below. * * #### Computing the hash * The process for computing a block hash is somewhat complex, and involves * creating a "virtual" merkle tree to obtain the root merkle hash of * that virtual tree.
- * The merkle tree SHALL have a 4 part structure with 2 internal nodes, - * structured in a strictly binary tree. - * - The merkle tree root SHALL be the parent of both - * internal nodes. - * 1. The first "internal" node SHALL be the parent of the - * two "left-most" nodes. - * 1. The first leaf MUST be the previous block hash, and is a - * single 48-byte value. - * 1. The second leaf MUST be the root of a, strictly binary, merkle tree - * composed of all "input" block items in the block.
- * Input items SHALL be transactions, system transactions, - * and events.
- * Leaf nodes in this subtree SHALL be ordered in the same order - * that the block items are encountered in the stream. - * 1. The second "internal" node SHALL be the parent of the two - * "right-most" nodes. - * 1. The third leaf MUST be the root of a, strictly binary, merkle tree - * composed of all "output" block items in the block.
- * Output items SHALL be transaction result, transaction - * output, and state changes.
- * Leaf nodes in this subtree SHALL be ordered in the same order that - * the block items are encountered in the stream. - * 1. The fourth leaf MUST be the merkle tree root hash for network state - * at the start of the block, and is a single 48-byte value. - * - The block hash SHALL be the hash calculated for the root of this merkle + * The merkle tree SHALL have the following structure: + * - The merkle tree SHALL be structured as a strictly binary tree. + * - The merkle tree root SHALL have a "left-most" node containing the + * block's beginning timestamp. + * - The merkle tree root SHALL have a "right-most" subtree comprised of 16 + * leaves and associated internal nodes. + * - The first four "levels" of the "right-most" subtree SHALL be internal + * nodes, comprising 15 in total. + * - The fifth level of the "right-most" subtree SHALL be comprised of 16 + * components (detailed below). + * The "right-most" subtree's 16 components SHALL be, in order (from left + * to right): + * 1. A leaf containing the previous block's root hash + * 2. A sub-tree of "incrementally-collapsed" block hashes, ranging from + * the genesis block to the block immediately preceding the current + * block number, strictly ordered by block number + * 3. A leaf containing the hash of the state merkle tree at the start of + * the block + * 4. The root of a, strictly binary, merkle tree composed of all + * consensus header items in the block. Leaf nodes in this subtree SHALL + * be ordered in the same order that the block items are encountered in + * the stream. + * 5. The root of a, strictly binary, merkle tree composed of all input + * items in the block. Input items SHALL be transactions, system + * transactions, and events. Leaf nodes in this subtree SHALL be + * ordered in the same order that the block items are encountered in + * the stream. + * 6. The root of a, strictly binary, merkle tree composed of all output + * items in the block. Output items SHALL be transaction result or + * transaction output items. Leaf nodes in this subtree SHALL be ordered + * in the same order that the block items are encountered in the stream. + * 7. The root of a, strictly binary, merkle tree composed of all state + * changes in the block. State change items SHALL be transaction result or + * transaction output items. Leaf nodes in this subtree SHALL be ordered + * in the same order that the block items are encountered in the stream. + * 8. The root of a, strictly binary, merkle tree composed of all trace + * data items in the block. Trace data items SHALL be transaction result + * or transaction output items. Leaf nodes in this subtree SHALL be + * ordered in the same order that the block items are encountered in + * the stream. + * 9-16: Empty leaf nodes reserved for future use (SHALL be set to the + * "null"). Each of branches 4-8 SHALL (individually) be "complete" binary + * merkle trees, with nodes that would otherwise be missing replaced by a + * "null" hash leaf. + * For the "right-most" subtree's internal nodes: + * - Level four's internal nodes SHALL be the parents of the (ordered) + * 16 leaves described above + * - Level three's internal nodes SHALL be the parents of level four's + * (ordered) internal nodes + * - Level two's internal nodes SHALL be the parents of level three's + * (ordered) internal nodes + * - Level one's internal node SHALL be the (single) parent of level two's + * (ordered) internal nodes + * The block hash SHALL be the hash calculated for the root of this merkle * tree. * - The hash algorithm used SHALL be the algorithm specified in the * corresponding block header. - * - * The "inputs" and "outputs" subtrees SHALL be "complete" binary merkle trees, - * with nodes that would otherwise be missing replaced by a "null" hash - * leaf. */ message BlockProof { - /** - * The block this proof secures.
- * We provide this because a proof for a future block can be used to prove - * the state of the ledger at that block and the blocks before it.
- *

- * This value SHOULD match the block number of the current block, - * under normal operation. - */ - uint64 block = 1; + /** + * The block this proof secures.
+ * We provide this because a proof for a future block can be used to prove + * the state of the ledger at that block and the blocks before it.
+ *

+ * This value SHOULD match the block number of the current block, + * under normal operation. + */ + uint64 block = 1; - /** - * A set of hash values along with ordering information.
- * This list of hash values form the set of sibling hash values needed to - * correctly reconstruct the parent hash, and all hash values "above" that - * hash in the merkle tree. - *

- * A Block proof can be constructed by combining the sibling hashes for - * a previous block hash and sibling hashes for each entry "above" that - * node in the merkle tree of a block proof that incorporates that previous - * block hash. This form of block proof may be used to prove a chain of - * blocks when one or more older blocks is missing the original block - * proof that signed the block's merkle root directly. - *

- * This list MUST be ordered from the sibling of the node that contains - * this block's root node hash, and continues up the merkle tree to the - * root hash of the signed block proof. - *

- * If this block proof has a "direct" signature, then this list MUST be - * empty.
- * If this list is not empty, then this block proof MUST be verified by - * first constructing the "block" merkle tree and computing the root hash - * of that tree, then combining that hash with the values in this list, - * paying attention to the first/second sibling ordering, until the root - * merkle hash is produced from the last pair of sibling hashes. That - * "secondary" root hash MUST then be verified using - * the value of `block_signature`. - */ - repeated MerkleSiblingHash sibling_hashes = 2; + /** + * A set of hash values along with ordering information.
+ * This list of hash values form the set of sibling hash values needed to + * correctly reconstruct the parent hash, and all hash values "above" that + * hash in the merkle tree. + *

+ * A Block proof can be constructed by combining the sibling hashes for + * a previous block hash and sibling hashes for each entry "above" that + * node in the merkle tree of a block proof that incorporates that previous + * block hash. This form of block proof may be used to prove a chain of + * blocks when one or more older blocks is missing the original block + * proof that signed the block's merkle root directly. + *

+ * This list MUST be ordered from the sibling of the node that contains + * this block's root node hash, and continues up the merkle tree to the + * root hash of the signed block proof. + *

+ * If this block proof has a "direct" signature, then this list MUST be + * empty.
+ * If this list is not empty, then this block proof MUST be verified by + * first constructing the "block" merkle tree and computing the root hash + * of that tree, then combining that hash with the values in this list, + * paying attention to the first/second sibling ordering, until the root + * merkle hash is produced from the last pair of sibling hashes. That + * "secondary" root hash MUST then be verified using + * the value of `block_signature`. + */ + repeated MerkleSiblingHash sibling_hashes = 2; - /** - * The hinTS key that this signature verifies under; a stream consumer should - * only use this key after first checking the chain of trust proof. - */ - bytes verification_key = 3; + /** + * The hinTS key that this signature verifies under; a stream consumer should + * only use this key after first checking the chain of trust proof. + */ + bytes verification_key = 3; - /** - * Proof the hinTS verification key is in the chain of trust extending - * from the network's ledger id. - */ - ChainOfTrustProof verification_key_proof = 4; + /** + * Proof the hinTS verification key is in the chain of trust extending + * from the network's ledger id. + */ + ChainOfTrustProof verification_key_proof = 4; - /** - * TODO - */ - oneof proof { - TssSignedBlockProof signed_block_proof = 5; - StateProof block_state_proof = 6; - SignedRecordFileProof signed_record_file_proof = 7; - } + /** + * The proof contents verifying the block's merkle root hash.
+ * This is a `oneof` field that MAY contain one several types of + * proofs. + */ + oneof proof { + /** + * A TSS signature over the block's merkle root hash.
+ * This signature SHALL be produced by a threshold signature scheme + * that allows multiple nodes to contribute partial signatures that + * can be aggregated into a single signature. This field MUST be + * used when the current block is signed directly by the consensus + * nodes with a TSS signature; otherwise it MUST be empty. + */ + TssSignedBlockProof signed_block_proof = 5; + /** + * A proof of the block merkle tree's contents. This proof SHALL + * contain the information necessary to validate the previous block's + * hash, along with any information necessary to validate the current + * block's hash. + */ + StateProof block_state_proof = 6; + /** + * A proof consisting of RSA signatures from consensus nodes.
+ * This proof type exists for backward compatibility with blocks that + * wrap historical record files.
+ * This field MUST be set when the block wraps a record file signed by + * individual RSA signatures from consensus nodes; otherwise it MUST be + * empty. + */ + SignedRecordFileProof signed_record_file_proof = 7; + } } /** @@ -148,23 +193,23 @@ message BlockProof { * a node of interest up to the root of the tree. */ message MerkleSiblingHash { - /** - * A flag for the position of this sibling. - *

- * If this is set then this sibling MUST be the first hash in the pair of - * sibling hashes of a binary merkle tree.
- * If this is unset, then this sibling MUST be the second hash in the pair - * of sibling hashes of a binary merkle tree. - */ - bool is_first = 1; + /** + * A flag for the position of this sibling. + *

+ * If this is set then this sibling MUST be the first hash in the pair of + * sibling hashes of a binary merkle tree.
+ * If this is unset, then this sibling MUST be the second hash in the pair + * of sibling hashes of a binary merkle tree. + */ + bool is_first = 1; - /** - * A byte array of a sibling hash.
- * This is the hash for the sibling at this point in the merkle tree. - *

- * The algorithm for this hash SHALL match the algorithm for the block that - * contains this sibling.
- * This SHALL contain the raw (e.g.) 384 bits (48 bytes) of the hash value. - */ - bytes sibling_hash = 2; + /** + * A byte array of a sibling hash.
+ * This is the hash for the sibling at this point in the merkle tree. + *

+ * The algorithm for this hash SHALL match the algorithm for the block that + * contains this sibling.
+ * This SHALL contain the raw (e.g.) 384 bits (48 bytes) of the hash value. + */ + bytes sibling_hash = 2; } \ No newline at end of file diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/chain_of_trust_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/chain_of_trust_proof.proto index 837d8802f339..9488fc13e06b 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/chain_of_trust_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/chain_of_trust_proof.proto @@ -1,5 +1,7 @@ /** - * TODO + * Proof that some data belongs to the network's chain of trust, starting from + * the genesis network whose address book hash formed the ledger id. (In the + * current system, this data is always a hinTS verification key.) * * ### Keywords * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", @@ -17,26 +19,21 @@ option java_package = "com.hedera.hapi.block.stream.protoc"; // <<>> This comment is special code for setting PBJ Compiler java package option java_multiple_files = true; -/** - * Proof that some data belongs to the network's chain of trust, starting from - * the genesis network whose address book hash formed the ledger id. (In the - * current system, this data is always a hinTS verification key.) - */ message ChainOfTrustProof { - oneof proof { - /** - * If there is not yet a SNARK proving the chain of trust from ledger id to - * the hinTS verification key, the explicit list of Schnorr signatures on - * the concatenation of the ledger id and genesis hinTS verification key - * that serve as witnesses for the SNARK prover algorithm. - */ - NodeSignatures node_signatures = 1; - /** - * If known, a ZK-compressed SNARK proof proving the chain of trust from - * the ledger id to this hinTS verification key. - */ - bytes wraps_proof = 2; - } + oneof proof { + /** + * If there is not yet a SNARK proving the chain of trust from ledger id to + * the hinTS verification key, the explicit list of Schnorr signatures on + * the concatenation of the ledger id and genesis hinTS verification key + * that serve as witnesses for the SNARK prover algorithm. + */ + NodeSignatures node_signatures = 1; + /** + * If known, a ZK-compressed SNARK proof proving the chain of trust from + * the ledger id to this hinTS verification key. + */ + bytes wraps_proof = 2; + } } /** @@ -48,7 +45,7 @@ message ChainOfTrustProof { * until the first recursive proof is available. */ message NodeSignatures { - repeated NodeSignature node_signatures = 1; + repeated NodeSignature node_signatures = 1; } /** @@ -56,12 +53,12 @@ message NodeSignatures { * message. */ message NodeSignature { - /** - * The node id of the signer. - */ - uint64 node_id = 1; - /** - * The signature. - */ - bytes signature = 2; + /** + * The node id of the signer. + */ + uint64 node_id = 1; + /** + * The signature. + */ + bytes signature = 2; } \ No newline at end of file diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto index 76a36f1cf0d4..59c8e2d7d58d 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/block_footer.proto @@ -1,6 +1,9 @@ /** * # Block Footer - * TODO + * A collection of hashes of sub parts of the blocks top fixed merkle + * tree that are needed to compute the blocks root hash. These are the + * hashes of the first 3 nodes across the bottom of the block fixed + * merkle tree in field order. * * ### Keywords * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", @@ -18,24 +21,20 @@ option java_package = "com.hedera.hapi.block.stream.output.protoc"; // <<>> This comment is special code for setting PBJ Compiler java package option java_multiple_files = true; -/** - * A collection of hashes of sub parts of the blocks top fixed merkle tree that are needed to compute the - * blocks root hash. These are the hashes of the first 3 nodes across the bottom of the block fixed merkle - * tree in field order. - */ message BlockFooter { + /** The root hash of the block, for the previous block to the one this + * footer belongs to. + */ + bytes previous_block_root_hash = 1; - /** The root hash of the block, for the previous block to the one this footer belongs to. - */ - bytes previous_block_root_hash = 1; - - /** - * The root hash of a merkle tree containg the root hashes of all block from block zero up to but not - * including this current block. - */ - bytes root_hash_of_all_block_hashes_tree = 2; + /** + * The root hash of a merkle tree containg the root hashes of all block + * from block zero up to but not including this current block. + */ + bytes root_hash_of_all_block_hashes_tree = 2; - /** The root hash of the state merkle tree for the version of state at the beginning of the current block - */ - bytes start_of_block_state_root_hash = 3; + /** The root hash of the state merkle tree for the version of state at + * the beginning of the current block. + */ + bytes start_of_block_state_root_hash = 3; } diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/state_changes.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/state_changes.proto index 138ad1766b8d..a0585e240920 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/state_changes.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/output/state_changes.proto @@ -473,6 +473,11 @@ enum StateIdentifier { */ STATE_ID_LAMBDA_STORAGE = 52; + /** + * A state identifier fro account to node relation. + */ + STATE_ID_ACCOUNT_NODE_REL = 53; + /** * A state identifier for the round receipts queue. Queue state. */ @@ -1074,6 +1079,11 @@ message MapChangeValue { * An EVM hook's state. */ com.hedera.hapi.node.state.hooks.EvmHookState evm_hook_state_value = 29; + + /** + * A node identifier. + */ + com.hedera.hapi.platform.state.NodeId node_id_value = 30; } } diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto index 5ec8e17d3081..6e17f545484d 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto @@ -1,5 +1,12 @@ /** - * TODO + * A state proof that cryptographically proves one of more nodes in the block + * merkle tree.
+ * Those nodes can represent state, block items or anything else + * in the tree. + * + * All state proofs MUST include the `TimeStamp` leaf at the top of the + * block merkle tree so that the point in time that applies to this proof is + * also proven. * * ### Keywords * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", @@ -21,66 +28,72 @@ import "services/timestamp.proto"; import "block/stream/record_file_item.proto"; import "services/state/blockstream/merkle_leaf.proto"; -/** - * TODO - */ message StateProof { - - /** - * TODO - */ - repeated MerklePath paths = 1; - - oneof proof { - /** - * TODO - */ - TssSignedBlockProof signed_block_proof = 2; - /** - * TODO + * The merkle paths that prove the nodes in the block merkle tree. + * This field SHALL contain, in the following order: + * - The merkle path from the previous block root (the "left-most" + * node on level 5) to it's (single) internal node parent (on level + * 1, adjacent to the block's consensus timestamp). + * - The merkle path from the consensus timestamp leaf to the root. + * - The merkle path of the root of the entire block merkle tree. */ - SignedRecordFileProof signed_record_file_proof = 4; - } + repeated MerklePath paths = 1; + + oneof proof { + /** + * A proof containing a TSS signature for a single block. This field + * SHALL contain the proof of a block with a block number greater + * than or equal to the current block; otherwise this field MUST be + * omitted. + */ + TssSignedBlockProof signed_block_proof = 2; + + /** + * A proof containing RSA signatures from consensus nodes for a record + * file. This field MUST contain the proof of the record file only if + * the state proof is being provided for a record file; otherwise it + * MUST be omitted. + */ + SignedRecordFileProof signed_record_file_proof = 4; + } } /** - * TODO + * A proof containing a TSS signature for a single block. */ message TssSignedBlockProof { - - /** - * A TSS signature for one block.
- * This is a single signature representing the collection of partial - * signatures from nodes holding strictly greater than 2/3 of the - * current network "weight" in aggregate. The signature is produced by - * cryptographic "aggregation" of the partial signatures to produce a - * single signature that can be verified with the network public key, - * but could not be produced by fewer nodes than required to meet the - * threshold for network stake "weight". - *

- * This message MUST make use of a threshold signature scheme like `BLS` - * which provides the necessary cryptographic guarantees.
- * This signature SHALL use a TSS signature to provide a single signature - * that represents the consensus signature of consensus nodes.
- * The exact subset of nodes that signed SHALL neither be known nor - * tracked, but it SHALL be cryptographically verifiable that the - * threshold was met if the signature itself can be validated with - * the network public key (a.k.a `LedgerID`). - */ - bytes block_signature = 1; + /** + * A TSS signature for one block.
+ * This is a single signature representing the collection of partial + * signatures from nodes holding strictly greater than 2/3 of the + * current network "weight" in aggregate. The signature is produced by + * cryptographic "aggregation" of the partial signatures to produce a + * single signature that can be verified with the network public key, + * but could not be produced by fewer nodes than required to meet the + * threshold for network stake "weight". + *

+ * This message MUST make use of a threshold signature scheme like `BLS` + * which provides the necessary cryptographic guarantees.
+ * This signature SHALL use a TSS signature to provide a single signature + * that represents the consensus signature of consensus nodes.
+ * The exact subset of nodes that signed SHALL neither be known nor + * tracked, but it SHALL be cryptographically verifiable that the + * threshold was met if the signature itself can be validated with + * the network public key (a.k.a `LedgerID`). + */ + bytes block_signature = 1; } /** - * TODO + * A proof containing RSA signatures from consensus nodes for a record file. */ message SignedRecordFileProof { - - /** - * A collection of RSA signatures from consensus nodes.
- * These signatures validate the hash of the record_file_contents field. - */ - repeated RecordFileSignature record_file_signatures = 1; + /** + * A collection of RSA signatures from consensus nodes.
+ * These signatures validate the hash of the record_file_contents field. + */ + repeated RecordFileSignature record_file_signatures = 1; } /** @@ -98,47 +111,47 @@ message SignedRecordFileProof { */ message MerklePath { - oneof content { + oneof content { + /** + * Optional leaf, if this path starts from a leaf + */ + com.hedera.hapi.node.state.blockstream.MerkleLeaf leaf = 1; + + /** + * Optional hash content for a path with no leaf that hashes paths below + * it. Needed for cases like proving a block root hash from another block. + */ + bytes hash = 2; + } + /** - * Optional leaf, if this path starts from a leaf + * Array of sibling nodes ordered from bottom of tree to top */ - com.hedera.hapi.node.state.blockstream.MerkleLeaf leaf = 1; + repeated SiblingNode siblings = 3; /** - * Optional hash content for a path with no leaf that hashes paths below - * it. Needed for cases like proving a block root hash from another block. + * The next parent path of this path going up the tree. Expressed as an index + * into the array of MerklePaths in the StatePoof. For example 0 being first + * in list etc. If this is the root path then the value is UINT32_MAX + * (this is `-1` in Java; 0xFFFFFFFF). */ - bytes hash = 2; - } - - /** - * Array of sibling nodes ordered from bottom of tree to top - */ - repeated SiblingNode siblings = 3; - - /** - * The next parent path of this path going up the tree. Expressed as an index - * into the array of MerklePaths in the StatePoof. For example 0 being first - * in list etc. If this is the root path then the value is UINT32_MAX - * (this is `-1` in Java; 0xFFFFFFFF). - */ - uint32 nextPathIndex = 4; + uint32 nextPathIndex = 4; } /** * Sibling Node, this represents the hash of a sibling node in a MerklePath */ message SiblingNode { - /** - * True when this sibling is on the left of the merkle path, False if on - * the right side of the merkle path - */ - bool is_left = 1; - - /** - * The hash of the sibling node that can be combined with the merkle path - * computed hashes as the proof is validated. - */ - bytes hash = 2; + /** + * True when this sibling is on the left of the merkle path, False if on + * the right side of the merkle path + */ + bool is_left = 1; + + /** + * The hash of the sibling node that can be combined with the merkle path + * computed hashes as the proof is validated. + */ + bytes hash = 2; } diff --git a/hedera-node/hapi-utils/src/main/java/com/hedera/node/app/hapi/utils/blocks/BlockStreamUtils.java b/hedera-node/hapi-utils/src/main/java/com/hedera/node/app/hapi/utils/blocks/BlockStreamUtils.java index a38df18f4981..5fd0e5d466b4 100644 --- a/hedera-node/hapi-utils/src/main/java/com/hedera/node/app/hapi/utils/blocks/BlockStreamUtils.java +++ b/hedera-node/hapi-utils/src/main/java/com/hedera/node/app/hapi/utils/blocks/BlockStreamUtils.java @@ -24,6 +24,7 @@ public static String stateNameOf(final int stateId) { return switch (StateIdentifier.fromProtobufOrdinal(stateId)) { case UNKNOWN -> throw new IllegalArgumentException("Unknown state identifier"); case STATE_ID_NODES -> "AddressBookService.NODES"; + case STATE_ID_ACCOUNT_NODE_REL -> "AddressBookService.ACCOUNT_NODE_REL"; case STATE_ID_BLOCKS -> "BlockRecordService.BLOCKS"; case STATE_ID_RUNNING_HASHES -> "BlockRecordService.RUNNING_HASHES"; case STATE_ID_BLOCK_STREAM_INFO -> "BlockStreamService.BLOCK_STREAM_INFO"; @@ -184,6 +185,7 @@ public static Object mapValueFor(@NonNull final MapChangeValue mapChangeValue) { case HISTORY_SIGNATURE_VALUE -> mapChangeValue.historySignatureValue(); case PROOF_KEY_SET_VALUE -> mapChangeValue.proofKeySetValue(); case EVM_HOOK_STATE_VALUE -> mapChangeValue.evmHookStateValueOrThrow(); + case NODE_ID_VALUE -> mapChangeValue.nodeIdValueOrThrow(); }; } From ff3e0f76e0ebfc11e4a9a6c5935b8b4210d6d269 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Tue, 28 Oct 2025 02:32:33 -0600 Subject: [PATCH 33/63] Docs Signed-off-by: Matt Hess --- .../state/blockstream/block_stream_info.proto | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto index 3d7e74b5e196..da13e87a04ab 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto @@ -102,9 +102,13 @@ message BlockStreamInfo { uint32 num_preceding_state_changes_items = 7; /** - * TODO - */ - repeated bytes intermediate_previous_block_root_hashes = 8; + * A concatenation of SHA2-384 hash values.
+ * This is the "rightmost" values of the "output" subtree. + *

+ * The subtree containing these hashes SHALL be constructed from all "output" + * `BlockItem`s in this block that _precede_ the update to this singleton. + */ + repeated bytes rightmost_preceding_state_changes_tree_hashes = 8; /** * A block-end consensus time stamp. @@ -157,15 +161,6 @@ message BlockStreamInfo { */ bytes output_item_root_hash = 15; - /** - * A SHA2-384 hash value.
- * This is the penultimate hash of the "input" subtree for - * this block. The final hash of the "state change" subtree MUST - * be calculated immediately after this block stream info object - * is persisted to state, and its accompanying state change emitted. - */ - bytes penultimate_state_change_item_root_hash = 16; - /** * A SHA2-384 hash value.
* This is the final hash of the "trace data" subtree for this block. @@ -173,7 +168,16 @@ message BlockStreamInfo { bytes trace_data_root_hash = 17; /** - * TODO + * The time stamp at which the first transaction was handled in + * this block. */ proto.Timestamp block_start_consensus_timestamp = 18; + + /** + * The intermediate hashes needed for branch 2 in the block merkle + * tree structure. These hashes SHALL include all block root hashes + * needed to construct branch 2's final state at the end of the + * previous block + */ + repeated bytes intermediate_previous_block_root_hashes = 19; } From 01aecd25e5da52c39c1c3faeae11c8303b94bd48 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Tue, 28 Oct 2025 10:15:28 -0600 Subject: [PATCH 34/63] Fixes Signed-off-by: Matt Hess --- .../blocks/impl/BlockStreamManagerImpl.java | 133 ++++++++++++------ .../impl/IncrementalStreamingHasher.java | 36 +++-- 2 files changed, 104 insertions(+), 65 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 5ce446459035..d831f1e2cf56 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -9,6 +9,7 @@ import static com.hedera.node.app.blocks.BlockStreamManager.PendingWork.NONE; import static com.hedera.node.app.blocks.BlockStreamManager.PendingWork.POST_UPGRADE_WORK; import static com.hedera.node.app.blocks.impl.BlockImplUtils.appendHash; +import static com.hedera.node.app.blocks.impl.ConcurrentStreamingTreeHasher.rootHashFrom; import static com.hedera.node.app.blocks.impl.streaming.FileBlockItemWriter.blockDirFor; import static com.hedera.node.app.blocks.impl.streaming.FileBlockItemWriter.cleanUpPendingBlock; import static com.hedera.node.app.blocks.impl.streaming.FileBlockItemWriter.loadContiguousPendingBlocks; @@ -148,17 +149,14 @@ public class BlockStreamManagerImpl implements BlockStreamManager { private BlockItemWriter writer; private Instant firstConsensusTimeOfCurrentBlock; - // block merkle tree - private IncrementalStreamingHasher previousBlockHashes; // ALL previous hashes, but streaming-collapsed - // TODO: still needed? + // Block merkle subtrees and leaves + private IncrementalStreamingHasher previousBlockHashes; private Bytes stateHashAtStartOfBlock; - private StreamingTreeHasher consensusHeaderHasher; private StreamingTreeHasher inputTreeHasher; private StreamingTreeHasher outputTreeHasher; private StreamingTreeHasher stateChangesHasher; private StreamingTreeHasher traceDataHasher; - // end block merkle tree private BlockStreamManagerTask worker; private final boolean hintsEnabled; @@ -264,7 +262,7 @@ public BlockStreamManagerImpl( .withDescription("Number of blocks closed with indirect proofs")); log.info( - "Initialized BlockStreamManager from round {} with end-of-round hash {}", + "Initialized BlockStreamManager from round {} with end-of-round state hash {}", lastRoundOfPrevBlock, hashFuture.isDone() ? hashFuture.join().toHex() : ""); } @@ -289,32 +287,40 @@ public void initBlockTrees(@NonNull final State state, @Nullable final Bytes las blockStreamInfo.trailingBlockHashes(), blockStreamInfo.blockNumber() - 1, blockStreamInfo.blockNumber() - 1); - requireNonNull(prevBlockHash); // Branch 2 - final var prevBlocksHasher = blockStreamInfo.intermediatePreviousBlockRootHashes().stream() + final var prevBlocksIntermediateHashes = blockStreamInfo.intermediatePreviousBlockRootHashes().stream() .map(Bytes::toByteArray) .toList(); - previousBlockHashes = new IncrementalStreamingHasher(prevBlocksHasher); - final var allPrevBlocksHash = Bytes.wrap(new IncrementalStreamingHasher(prevBlocksHasher).computeRootHash()); + previousBlockHashes = + new IncrementalStreamingHasher(CommonUtils.sha384DigestOrThrow(), prevBlocksIntermediateHashes); + final var allPrevBlocksHash = Bytes.wrap(previousBlockHashes.computeRootHash()); + // Branch 3: Retrieve the previous block's starting state hash (not done right here, just part of the calculated // last block hash below) - // Branches 4-8 - resetBlockOnlyBranches(); // We have to calculate the final hash of the previous block's state changes subtree because only the - // penultimate state hash is in the block stream info object - final var lastBlockPenultimateStateHash = blockStreamInfo.penultimateStateChangeItemRootHash(); - // Reconstruct the final state change block item that would have been emitted + // penultimate state hash is in the block stream info object (constructed from numPrecedingStateChangesItems and + // rightmostPrecedingStateChangesTreeHashes) + final var penultimateStateChangesTreeStatus = new StreamingTreeHasher.Status( + blockStreamInfo.numPrecedingStateChangesItems(), + blockStreamInfo.rightmostPrecedingStateChangesTreeHashes()); + + // Reconstruct the final state change block item that would have been emitted by the previous block final var lastBlockFinalStateChange = StateChange.newBuilder() .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal()) .singletonUpdate(SingletonUpdateChange.newBuilder() .blockStreamInfoValue(blockStreamInfo) .build()) .build(); - final var changeBytes = StateChange.PROTOBUF.toBytes(lastBlockFinalStateChange); - // Combine the penultimate hash and the hash of the reconstructed state change item to produce the previous - // block's final state changes hash - final var lastBlockFinalStateChangesHash = BlockImplUtils.combine(lastBlockPenultimateStateHash, changeBytes); + final var lastStateChanges = BlockItem.newBuilder() + .stateChanges(new StateChanges(blockStreamInfo.blockEndTime(), List.of(lastBlockFinalStateChange))) + .build(); + // Hash the reconstructed (final) state changes block item + final var lastLeafHash = noThrowSha384HashOf(BlockItem.PROTOBUF.toBytes(lastStateChanges)); + + // Combine the penultimate tree status and the hash of the reconstructed state change item to produce the + // previous block's final state changes hash + final var lastBlockFinalStateChangesHash = rootHashFrom(penultimateStateChangesTreeStatus, lastLeafHash); final var calculatedLastBlockHash = Optional.ofNullable(lastBlockHash) .orElseGet(() -> BlockStreamManagerImpl.combine( @@ -345,6 +351,11 @@ public void startRound(@NonNull final Round round, @NonNull final State state) { // In case we hash this round, include a future for the end-of-round state hash endRoundStateHashes.put(round.getRoundNum(), new CompletableFuture<>()); + if (lastRoundOfPrevBlock > 0) { + stateHashAtStartOfBlock = + endRoundStateHashes.get(lastRoundOfPrevBlock).join(); + } + // Writer will be null when beginning a new block if (writer == null) { writer = writerSupplier.get(); @@ -360,7 +371,7 @@ public void startRound(@NonNull final Round round, @NonNull final State state) { lifecycle.onOpenBlock(state); - resetBlockOnlyBranches(); + resetContainedSubtrees(); blockNumber = blockStreamInfo.blockNumber() + 1; if (hintsEnabled && !hasCheckedForPendingBlocks) { @@ -519,7 +530,6 @@ public boolean endRound(@NonNull final State state, final long roundNum) { // Branch 2 final var prevBlockRootsHash = Bytes.wrap(previousBlockHashes.computeRootHash()); // Branch 3: blockStartStateHash - // Calculate hashes for branches 4-8 final Map computedHashes = new HashMap<>(); final var future = CompletableFuture.allOf( @@ -531,10 +541,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { inputTreeHasher.rootHash().thenAccept(b -> computedHashes.put(SubMerkleTree.INPUT_ITEMS_TREE, b)), // Branch 6 outputTreeHasher.rootHash().thenAccept(b -> computedHashes.put(SubMerkleTree.OUTPUT_ITEMS_TREE, b)), - // Branch 7 - stateChangesHasher - .rootHash() - .thenAccept(b -> computedHashes.put(SubMerkleTree.STATE_CHANGE_ITEMS_TREE, b)), + // Branch 7 will be computed below after adding the final state change item // Branch 8 traceDataHasher .rootHash() @@ -547,9 +554,9 @@ public boolean endRound(@NonNull final State state, final long roundNum) { final var inputsHash = computedHashes.get(SubMerkleTree.INPUT_ITEMS_TREE); // Branch 6 final hash: final var outputsHash = computedHashes.get(SubMerkleTree.OUTPUT_ITEMS_TREE); - // Branch 7 (penultimate because there will be one more state change when the block stream info object is - // stored) - final var penultimateStateChangesHash = computedHashes.get(SubMerkleTree.STATE_CHANGE_ITEMS_TREE); + // Branch 7 (penultimate status only because there will be one more state change when the block stream info + // object is stored) + final var penultimateStateChangesTreeStatus = stateChangesHasher.status(); // Branch 8 final hash: final var traceDataHash = computedHashes.get(SubMerkleTree.TRACE_DATA_ITEMS_TREE); @@ -563,8 +570,8 @@ public boolean endRound(@NonNull final State state, final long roundNum) { blockHashManager.blockHashes(), inputsHash, blockStartStateHash, - stateChangesHasher.status().numLeaves(), - previousBlockHashes.intermediateHashingState(), + penultimateStateChangesTreeStatus.numLeaves(), + penultimateStateChangesTreeStatus.rightmostHashes(), lastUsedTime, pendingWork != POST_UPGRADE_WORK, version, @@ -572,9 +579,9 @@ public boolean endRound(@NonNull final State state, final long roundNum) { asTimestamp(lastTopLevelTime), consensusHeaderHash, outputsHash, - penultimateStateChangesHash, traceDataHash, - asTimestamp(firstConsensusTimeOfCurrentBlock)); + asTimestamp(firstConsensusTimeOfCurrentBlock), + previousBlockHashes.intermediateHashingState()); blockStreamInfoState.put(newBlockStreamInfo); ((CommittableWritableStates) writableState).commit(); @@ -582,6 +589,8 @@ public boolean endRound(@NonNull final State state, final long roundNum) { worker.addItem(flushChangesFromListener(boundaryStateChangeListener)); worker.sync(); + final var stateChangesHash = stateChangesHasher.rootHash().join(); + // Reconstruct the final state change in order to calculate the final state change subtree hash final var blockStreamInfoChange = StateChange.newBuilder() .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal()) @@ -589,9 +598,9 @@ public boolean endRound(@NonNull final State state, final long roundNum) { .blockStreamInfoValue(newBlockStreamInfo) .build()) .build(); - final var changeBytes = StateChange.PROTOBUF.toBytes(blockStreamInfoChange); + final var hashedChangeBytes = noThrowSha384HashOf(StateChange.PROTOBUF.toBytes(blockStreamInfoChange)); // Combine the penultimate state change leaf with the final state change leaf - final var finalStateChangesHash = BlockImplUtils.combine(penultimateStateChangesHash, changeBytes); + final var finalStateChangesHash = BlockImplUtils.combine(stateChangesHash, hashedChangeBytes); final var rootAndSiblingHashes = combine( lastBlockHash, @@ -633,6 +642,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { // Update in-memory state to prepare for the next block lastBlockHash = finalBlockRootHash; + previousBlockHashes.addLeaf(lastBlockHash.toByteArray()); writer = null; // Special case when signing with hinTS and this is the freeze round; we have to wait @@ -906,9 +916,8 @@ protected boolean onExecute() { TRANSACTION_OUTPUT, STATE_CHANGES, ROUND_HEADER, - BLOCK_HEADER - // Also EndBlock? - -> { + BLOCK_HEADER, + TRACE_DATA -> { MessageDigest digest = sha384DigestOrThrow(); bytes.writeTo(digest); hash = ByteBuffer.wrap(digest.digest()); @@ -947,6 +956,7 @@ protected boolean onExecute() { } case TRANSACTION_OUTPUT, BLOCK_HEADER -> outputTreeHasher.addLeaf(hash); case STATE_CHANGES -> stateChangesHasher.addLeaf(hash); + case TRACE_DATA -> traceDataHasher.addLeaf(hash); case BLOCK_FOOTER, BLOCK_PROOF -> { // BlockFooter and BlockProof are not included in any merkle tree // They are metadata about the block, not part of the hashed content @@ -1125,7 +1135,11 @@ private BlockItem flushChangesFromListener(@NonNull final BoundaryStateChangeLis return BlockItem.newBuilder().stateChanges(stateChanges).build(); } - private void resetBlockOnlyBranches() { + /** + * Resets the subtree hashers for branches 4-8 to empty states. Since these subtrees only contain data specific to + * the current block, they need to be reset whenever a new block starts. + */ + private void resetContainedSubtrees() { // Branch 4 consensusHeaderHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); // Branch 5 @@ -1140,16 +1154,43 @@ private void resetBlockOnlyBranches() { private record RootAndSiblingHashes(Bytes blockRootHash, MerkleSiblingHash[] siblingHashes) {} + /** + * Combines the given branch hashes into a block root hash and sibling hashes for a pending proof. + * Since it's not known whether the pending proof will be directly signed, the sibling hashes + * required for an indirect proof are also computed. + * @return the block root hash and all possibly-required sibling hashes, ordered from bottom (the + * leaf level) to top (the root) + */ private static RootAndSiblingHashes combine( - final Bytes prevBlockHash, - final Bytes prevBlockRootsHash, - final Bytes startingStateHash, - final Bytes consensusHeaderHash, - final Bytes inputsHash, - final Bytes outputsHash, - final Bytes stateChangesHash, - final Bytes traceDataHash, + @Nullable final Bytes maybePrevBlockHash, + @Nullable final Bytes maybePrevBlockRootsHash, + @Nullable final Bytes maybeStartingStateHash, + @Nullable final Bytes maybeConsensusHeaderHash, + @Nullable final Bytes maybeInputsHash, + @Nullable final Bytes maybeOutputsHash, + @Nullable final Bytes maybeStateChangesHash, + @Nullable final Bytes maybeTraceDataHash, final Timestamp firstConsensusTimeOfCurrentBlock) { + final var prevBlockHash = + (maybePrevBlockHash != null && maybePrevBlockHash.length() > 0) ? maybePrevBlockHash : NULL_HASH; + final var prevBlockRootsHash = (maybePrevBlockRootsHash != null && maybePrevBlockRootsHash.length() > 0) + ? maybePrevBlockRootsHash + : NULL_HASH; + final var startingStateHash = (maybeStartingStateHash != null && maybeStartingStateHash.length() > 0) + ? maybeStartingStateHash + : NULL_HASH; + final var consensusHeaderHash = (maybeConsensusHeaderHash != null && maybeConsensusHeaderHash.length() > 0) + ? maybeConsensusHeaderHash + : NULL_HASH; + final var inputsHash = (maybeInputsHash != null && maybeInputsHash.length() > 0) ? maybeInputsHash : NULL_HASH; + final var outputsHash = + (maybeOutputsHash != null && maybeOutputsHash.length() > 0) ? maybeOutputsHash : NULL_HASH; + final var stateChangesHash = (maybeStartingStateHash != null && maybeStartingStateHash.length() > 0) + ? maybeStartingStateHash + : NULL_HASH; + final var traceDataHash = + (maybeTraceDataHash != null && maybeTraceDataHash.length() > 0) ? maybeTraceDataHash : NULL_HASH; + // Compute depth four hashes final var depth4Node1 = BlockImplUtils.combine(prevBlockHash, prevBlockRootsHash); final var depth4Node2 = BlockImplUtils.combine(startingStateHash, consensusHeaderHash); diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java index 0c8f8a245252..d597e51dc8fd 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java @@ -3,7 +3,6 @@ import com.hedera.pbj.runtime.io.buffer.Bytes; import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.LinkedList; import java.util.List; @@ -24,26 +23,16 @@ public class IncrementalStreamingHasher { /** A list to store intermediate hashes as we build the tree. */ private final LinkedList hashList = new LinkedList<>(); /** The count of leaves in the tree. */ - private int leafCount = 0; + private int leafCount; - /** Create a new StreamingHasher with an empty state. */ - public IncrementalStreamingHasher() { - try { - digest = MessageDigest.getInstance("SHA-384"); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); + /** Create a StreamingHasher with an existing intermediate hashing state. */ + public IncrementalStreamingHasher(final MessageDigest digest, List intermediateHashingState) { + if (digest == null) { + throw new IllegalArgumentException("digest must not be null"); } - } - - /** - * Create a StreamingHasher with an existing intermediate hashing state. - * This allows resuming hashing from a previous state. - * - * @param intermediateHashingState the intermediate hashing state - */ - public IncrementalStreamingHasher(List intermediateHashingState) { - this(); + this.digest = digest; this.hashList.addAll(intermediateHashingState); + this.leafCount = intermediateHashingState.size(); } /** @@ -67,9 +56,16 @@ public void addLeaf(byte[] data) { * Compute the Merkle tree root hash from the current state. This does not modify the internal state, so can be * called at any time and more leaves can be added afterward. * - * @return the Merkle tree root hash + * @return the Merkle tree root hash, or {@code Bytes.EMPTY} if no leaves exist */ public byte[] computeRootHash() { + if (hashList.isEmpty()) { + return Bytes.EMPTY.toByteArray(); + } + if (hashList.size() == 1) { + return hashList.getFirst(); + } + byte[] merkleRootHash = hashList.getLast(); for (int i = hashList.size() - 2; i >= 0; i--) { merkleRootHash = hashInternalNode(hashList.get(i), merkleRootHash); @@ -103,6 +99,7 @@ public int leafCount() { * @return the hash of the leaf node */ private byte[] hashLeaf(final byte[] leafData) { + digest.reset(); digest.update(LEAF_PREFIX); return digest.digest(leafData); } @@ -115,6 +112,7 @@ private byte[] hashLeaf(final byte[] leafData) { * @return the hash of the internal node */ private byte[] hashInternalNode(final byte[] firstChild, final byte[] secondChild) { + digest.reset(); digest.update(INTERNAL_NODE_PREFIX); digest.update(firstChild); return digest.digest(secondChild); From 34a643898a790a424363c0c9ee6929c44f908f64 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Tue, 28 Oct 2025 10:31:32 -0600 Subject: [PATCH 35/63] Test fix Signed-off-by: Matt Hess --- .../impl/streaming/BlockNodeCommunicationTestBase.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockNodeCommunicationTestBase.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockNodeCommunicationTestBase.java index eef9623bef80..7ed74648b45f 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockNodeCommunicationTestBase.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockNodeCommunicationTestBase.java @@ -5,6 +5,7 @@ import com.hedera.hapi.block.stream.BlockItem; import com.hedera.hapi.block.stream.BlockProof; +import com.hedera.hapi.block.stream.TssSignedBlockProof; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.block.stream.output.SingletonUpdateChange; import com.hedera.hapi.block.stream.output.StateChange; @@ -143,7 +144,9 @@ protected static BlockItem newBlockProofItem(final long blockNumber, final int b final BlockProof proof = BlockProof.newBuilder() .block(blockNumber) - .blockSignature(Bytes.wrap(array)) + .signedBlockProof(TssSignedBlockProof.newBuilder() + .blockSignature(Bytes.wrap(array)) + .build()) .build(); return BlockItem.newBuilder().blockProof(proof).build(); } From 2c18cb2e7989524d4eaf288f28718045529d9984 Mon Sep 17 00:00:00 2001 From: Neeharika-Sompalli Date: Fri, 31 Oct 2025 23:17:26 -0500 Subject: [PATCH 36/63] wip Signed-off-by: Neeharika-Sompalli --- .../state/blockstream/block_stream_info.proto | 2 + .../blocks/impl/BlockStreamManagerImpl.java | 56 +++++++------------ .../impl/IncrementalStreamingHasher.java | 9 +-- 3 files changed, 28 insertions(+), 39 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto index da13e87a04ab..0e211e18f0fb 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto @@ -180,4 +180,6 @@ message BlockStreamInfo { * previous block */ repeated bytes intermediate_previous_block_root_hashes = 19; + + uint64 intermediate_block_roots_leaf_count = 20; } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index d831f1e2cf56..d7398ee2cea4 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -151,7 +151,6 @@ public class BlockStreamManagerImpl implements BlockStreamManager { // Block merkle subtrees and leaves private IncrementalStreamingHasher previousBlockHashes; - private Bytes stateHashAtStartOfBlock; private StreamingTreeHasher consensusHeaderHasher; private StreamingTreeHasher inputTreeHasher; private StreamingTreeHasher outputTreeHasher; @@ -288,11 +287,14 @@ public void initBlockTrees(@NonNull final State state, @Nullable final Bytes las blockStreamInfo.blockNumber() - 1, blockStreamInfo.blockNumber() - 1); // Branch 2 - final var prevBlocksIntermediateHashes = blockStreamInfo.intermediatePreviousBlockRootHashes().stream() + final var prevBlocksIntermediateHashes = blockStreamInfo + .intermediatePreviousBlockRootHashes() + .stream() .map(Bytes::toByteArray) .toList(); - previousBlockHashes = - new IncrementalStreamingHasher(CommonUtils.sha384DigestOrThrow(), prevBlocksIntermediateHashes); + previousBlockHashes = new IncrementalStreamingHasher(CommonUtils.sha384DigestOrThrow(), + prevBlocksIntermediateHashes, + blockStreamInfo.intermediateBlockRootsLeafCount()); final var allPrevBlocksHash = Bytes.wrap(previousBlockHashes.computeRootHash()); // Branch 3: Retrieve the previous block's starting state hash (not done right here, just part of the calculated @@ -335,7 +337,7 @@ public void initBlockTrees(@NonNull final State state, @Nullable final Bytes las blockStreamInfo.blockStartConsensusTimestamp()) .blockRootHash()); requireNonNull(calculatedLastBlockHash); - initLastBlockHash(calculatedLastBlockHash); + this.lastBlockHash = calculatedLastBlockHash; } @Override @@ -351,11 +353,6 @@ public void startRound(@NonNull final Round round, @NonNull final State state) { // In case we hash this round, include a future for the end-of-round state hash endRoundStateHashes.put(round.getRoundNum(), new CompletableFuture<>()); - if (lastRoundOfPrevBlock > 0) { - stateHashAtStartOfBlock = - endRoundStateHashes.get(lastRoundOfPrevBlock).join(); - } - // Writer will be null when beginning a new block if (writer == null) { writer = writerSupplier.get(); @@ -393,9 +390,9 @@ public void startRound(@NonNull final Round round, @NonNull final State state) { .blockTimestamp(asTimestamp(blockTimestamp)) .hapiProtoVersion(hapiVersion); worker.addItem(BlockItem.newBuilder().blockHeader(header).build()); + firstConsensusTimeOfCurrentBlock = round.getConsensusTimestamp(); } consensusTimeLastRound = round.getConsensusTimestamp(); - firstConsensusTimeOfCurrentBlock = round.getConsensusTimestamp(); } /** @@ -528,10 +525,9 @@ public boolean endRound(@NonNull final State state, final long roundNum) { // Branch 1: lastBlockHash // Branch 2 - final var prevBlockRootsHash = Bytes.wrap(previousBlockHashes.computeRootHash()); // Branch 3: blockStartStateHash // Calculate hashes for branches 4-8 - final Map computedHashes = new HashMap<>(); + final Map computedHashes = new ConcurrentHashMap<>(); final var future = CompletableFuture.allOf( // Branch 4 consensusHeaderHasher @@ -566,7 +562,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { final var newBlockStreamInfo = new BlockStreamInfo( blockNumber, blockTimestamp(), - runningHashManager.latestHashes(), // lastBlockHash is stored here + runningHashManager.latestHashes(), blockHashManager.blockHashes(), inputsHash, blockStartStateHash, @@ -581,7 +577,8 @@ public boolean endRound(@NonNull final State state, final long roundNum) { outputsHash, traceDataHash, asTimestamp(firstConsensusTimeOfCurrentBlock), - previousBlockHashes.intermediateHashingState()); + previousBlockHashes.intermediateHashingState(), + previousBlockHashes.leafCount()); blockStreamInfoState.put(newBlockStreamInfo); ((CommittableWritableStates) writableState).commit(); @@ -591,27 +588,17 @@ public boolean endRound(@NonNull final State state, final long roundNum) { final var stateChangesHash = stateChangesHasher.rootHash().join(); - // Reconstruct the final state change in order to calculate the final state change subtree hash - final var blockStreamInfoChange = StateChange.newBuilder() - .stateId(STATE_ID_BLOCK_STREAM_INFO.protoOrdinal()) - .singletonUpdate(SingletonUpdateChange.newBuilder() - .blockStreamInfoValue(newBlockStreamInfo) - .build()) - .build(); - final var hashedChangeBytes = noThrowSha384HashOf(StateChange.PROTOBUF.toBytes(blockStreamInfoChange)); - // Combine the penultimate state change leaf with the final state change leaf - final var finalStateChangesHash = BlockImplUtils.combine(stateChangesHash, hashedChangeBytes); - + final var prevBlockRootsHash = Bytes.wrap(previousBlockHashes.computeRootHash()); final var rootAndSiblingHashes = combine( lastBlockHash, prevBlockRootsHash, - stateHashAtStartOfBlock, + blockStartStateHash, consensusHeaderHash, inputsHash, outputsHash, - finalStateChangesHash, + stateChangesHash, traceDataHash, - asTimestamp(firstConsensusTimeOfCurrentBlock)); + newBlockStreamInfo.blockStartConsensusTimestamp()); final var finalBlockRootHash = rootAndSiblingHashes.blockRootHash(); // Create BlockFooter with the three essential hashes: @@ -619,14 +606,13 @@ public boolean endRound(@NonNull final State state, final long roundNum) { // 1. previousBlockRootHash - Root hash of the previous block (N-1) .previousBlockRootHash(lastBlockHash) // 2. rootHashOfAllBlockHashesTree - RootStreaming tree of all block hashes 0..N-1 - .rootHashOfAllBlockHashesTree(finalBlockRootHash) + .rootHashOfAllBlockHashesTree(prevBlockRootsHash) // 3. startOfBlockStateRootHash - State hash at the beginning of current block .startOfBlockStateRootHash(blockStartStateHash) .build(); // Write BlockFooter to block stream (last item before BlockProof) - final var footerItem = - BlockItem.newBuilder().blockFooter(blockFooter).build(); + final var footerItem = BlockItem.newBuilder().blockFooter(blockFooter).build(); worker.addItem(footerItem); worker.sync(); @@ -1185,8 +1171,8 @@ private static RootAndSiblingHashes combine( final var inputsHash = (maybeInputsHash != null && maybeInputsHash.length() > 0) ? maybeInputsHash : NULL_HASH; final var outputsHash = (maybeOutputsHash != null && maybeOutputsHash.length() > 0) ? maybeOutputsHash : NULL_HASH; - final var stateChangesHash = (maybeStartingStateHash != null && maybeStartingStateHash.length() > 0) - ? maybeStartingStateHash + final var stateChangesHash = (maybeStateChangesHash != null && maybeStateChangesHash.length() > 0) + ? maybeStateChangesHash : NULL_HASH; final var traceDataHash = (maybeTraceDataHash != null && maybeTraceDataHash.length() > 0) ? maybeTraceDataHash : NULL_HASH; @@ -1222,7 +1208,7 @@ private static RootAndSiblingHashes combine( final var rootHash = BlockImplUtils.combine(depth1Node0, depth1Node1); return new RootAndSiblingHashes(rootHash, new MerkleSiblingHash[] { // Level 5 first sibling (right child) - new MerkleSiblingHash(false, prevBlockHash), + new MerkleSiblingHash(false, prevBlockRootsHash), // Level 4 first sibling (right child) new MerkleSiblingHash(false, depth4Node2), // Level 3 first sibling (right child) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java index d597e51dc8fd..bcd9d4d10047 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java @@ -23,16 +23,17 @@ public class IncrementalStreamingHasher { /** A list to store intermediate hashes as we build the tree. */ private final LinkedList hashList = new LinkedList<>(); /** The count of leaves in the tree. */ - private int leafCount; + private long leafCount; /** Create a StreamingHasher with an existing intermediate hashing state. */ - public IncrementalStreamingHasher(final MessageDigest digest, List intermediateHashingState) { + public IncrementalStreamingHasher(final MessageDigest digest, List intermediateHashingState, + final long leafCount) { if (digest == null) { throw new IllegalArgumentException("digest must not be null"); } this.digest = digest; this.hashList.addAll(intermediateHashingState); - this.leafCount = intermediateHashingState.size(); + this.leafCount = leafCount; } /** @@ -88,7 +89,7 @@ public List intermediateHashingState() { * * @return the number of leaves */ - public int leafCount() { + public long leafCount() { return leafCount; } From e859fed82e549312f2d5b54d0b870663e55467f8 Mon Sep 17 00:00:00 2001 From: Neeharika-Sompalli Date: Fri, 31 Oct 2025 23:32:38 -0500 Subject: [PATCH 37/63] probablyyy fixes? Signed-off-by: Neeharika-Sompalli --- .../com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index d7398ee2cea4..6a305398af19 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -338,6 +338,7 @@ public void initBlockTrees(@NonNull final State state, @Nullable final Bytes las .blockRootHash()); requireNonNull(calculatedLastBlockHash); this.lastBlockHash = calculatedLastBlockHash; + previousBlockHashes.addLeaf(calculatedLastBlockHash.toByteArray()); } @Override From 7acd96926d457a5324676fc996aabf1f08fbbba1 Mon Sep 17 00:00:00 2001 From: Neeharika-Sompalli Date: Fri, 31 Oct 2025 23:41:39 -0500 Subject: [PATCH 38/63] spotless Signed-off-by: Neeharika-Sompalli --- .../app/blocks/impl/BlockStreamManagerImpl.java | 15 +++++++-------- .../blocks/impl/IncrementalStreamingHasher.java | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 6a305398af19..6692936495ba 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -80,7 +80,6 @@ import java.time.Instant; import java.util.ArrayList; import java.util.EnumSet; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -287,14 +286,13 @@ public void initBlockTrees(@NonNull final State state, @Nullable final Bytes las blockStreamInfo.blockNumber() - 1, blockStreamInfo.blockNumber() - 1); // Branch 2 - final var prevBlocksIntermediateHashes = blockStreamInfo - .intermediatePreviousBlockRootHashes() - .stream() + final var prevBlocksIntermediateHashes = blockStreamInfo.intermediatePreviousBlockRootHashes().stream() .map(Bytes::toByteArray) .toList(); - previousBlockHashes = new IncrementalStreamingHasher(CommonUtils.sha384DigestOrThrow(), - prevBlocksIntermediateHashes, - blockStreamInfo.intermediateBlockRootsLeafCount()); + previousBlockHashes = new IncrementalStreamingHasher( + CommonUtils.sha384DigestOrThrow(), + prevBlocksIntermediateHashes, + blockStreamInfo.intermediateBlockRootsLeafCount()); final var allPrevBlocksHash = Bytes.wrap(previousBlockHashes.computeRootHash()); // Branch 3: Retrieve the previous block's starting state hash (not done right here, just part of the calculated @@ -613,7 +611,8 @@ public boolean endRound(@NonNull final State state, final long roundNum) { .build(); // Write BlockFooter to block stream (last item before BlockProof) - final var footerItem = BlockItem.newBuilder().blockFooter(blockFooter).build(); + final var footerItem = + BlockItem.newBuilder().blockFooter(blockFooter).build(); worker.addItem(footerItem); worker.sync(); diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java index bcd9d4d10047..bd9715793957 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java @@ -26,8 +26,8 @@ public class IncrementalStreamingHasher { private long leafCount; /** Create a StreamingHasher with an existing intermediate hashing state. */ - public IncrementalStreamingHasher(final MessageDigest digest, List intermediateHashingState, - final long leafCount) { + public IncrementalStreamingHasher( + final MessageDigest digest, List intermediateHashingState, final long leafCount) { if (digest == null) { throw new IllegalArgumentException("digest must not be null"); } From 3779cd1a627c5f551e4639a54f70020d2292a54c Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sat, 1 Nov 2025 01:56:59 -0600 Subject: [PATCH 39/63] Minor cleanup Signed-off-by: Matt Hess --- .../node/app/hapi/utils/CommonUtils.java | 11 ++++ .../node/app/hapi/utils/CommonUtilsTest.java | 21 +++++++ .../main/java/com/hedera/node/app/Hedera.java | 2 +- .../node/app/blocks/BlockStreamManager.java | 2 +- .../blocks/impl/BlockStreamManagerImpl.java | 61 ++++++++----------- 5 files changed, 61 insertions(+), 36 deletions(-) diff --git a/hedera-node/hapi-utils/src/main/java/com/hedera/node/app/hapi/utils/CommonUtils.java b/hedera-node/hapi-utils/src/main/java/com/hedera/node/app/hapi/utils/CommonUtils.java index 6a4f837ce11a..15f3f9e3d06b 100644 --- a/hedera-node/hapi-utils/src/main/java/com/hedera/node/app/hapi/utils/CommonUtils.java +++ b/hedera-node/hapi-utils/src/main/java/com/hedera/node/app/hapi/utils/CommonUtils.java @@ -6,6 +6,7 @@ import static com.hedera.node.app.hapi.utils.CommonPbjConverters.toPbj; import static java.lang.System.arraycopy; import static java.util.Objects.requireNonNull; +import static org.hiero.base.crypto.Cryptography.NULL_HASH; import com.google.common.annotations.VisibleForTesting; import com.google.common.primitives.Longs; @@ -22,6 +23,7 @@ import com.hederahashgraph.api.proto.java.TransactionBody; import com.hederahashgraph.api.proto.java.TransactionOrBuilder; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Instant; @@ -185,4 +187,13 @@ public static long clampedAdd(final long addendA, final long addendB) { return addendA > 0 ? Long.MAX_VALUE : Long.MIN_VALUE; } } + + /** + * Returns the given hash if it is non-null and non-empty; otherwise, returns {@code NULL_HASH} + * @param maybeHash the possibly null or empty hash + * @return the given hash or {@code NULL_HASH} if the given hash is null or empty + */ + public static Bytes inputOrNullHash(@Nullable final Bytes maybeHash) { + return (maybeHash != null && maybeHash.length() > 0) ? maybeHash : NULL_HASH.getBytes(); + } } diff --git a/hedera-node/hapi-utils/src/test/java/com/hedera/node/app/hapi/utils/CommonUtilsTest.java b/hedera-node/hapi-utils/src/test/java/com/hedera/node/app/hapi/utils/CommonUtilsTest.java index 1873c469a8c6..8274b5271674 100644 --- a/hedera-node/hapi-utils/src/test/java/com/hedera/node/app/hapi/utils/CommonUtilsTest.java +++ b/hedera-node/hapi-utils/src/test/java/com/hedera/node/app/hapi/utils/CommonUtilsTest.java @@ -49,6 +49,7 @@ import static com.hederahashgraph.api.proto.java.HederaFunctionality.UncheckedSubmit; import static com.hederahashgraph.api.proto.java.HederaFunctionality.UtilPrng; import static com.hederahashgraph.api.proto.java.ResponseType.ANSWER_ONLY; +import static org.hiero.base.crypto.Cryptography.NULL_HASH; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -59,6 +60,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.GeneratedMessage; import com.google.protobuf.InvalidProtocolBufferException; +import com.hedera.pbj.runtime.io.buffer.Bytes; import com.hederahashgraph.api.proto.java.AccountID; import com.hederahashgraph.api.proto.java.ConsensusCreateTopicTransactionBody; import com.hederahashgraph.api.proto.java.ConsensusDeleteTopicTransactionBody; @@ -311,4 +313,23 @@ void getExpectEvmAddress() { final var evmAddress = asEvmAddress(123L); assertArrayEquals(address, evmAddress); } + + @Test + void inputOrNullHashReturnsHash() { + final Bytes input = Bytes.wrap(new byte[] {1, 2, 3, 4, 5}); + final var result = CommonUtils.inputOrNullHash(input); + assertEquals(input, result); + } + + @Test + void inputOrNullHashReturnsNullHash() { + final var result = CommonUtils.inputOrNullHash(null); + assertEquals(NULL_HASH.getBytes(), result); + } + + @Test + void inputOrNullHashReturnsNullHashForEmpty() { + final var result = CommonUtils.inputOrNullHash(Bytes.EMPTY); + assertEquals(NULL_HASH.getBytes(), result); + } } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java index 66132687a4be..9b711603bfc9 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/Hedera.java @@ -1241,7 +1241,7 @@ private void initializeDagger(@NonNull final State state, @NonNull final InitTri final var lastBlockHash = (trigger == GENESIS) ? ZERO_BLOCK_HASH : blockStreamService.migratedLastBlockHash().orElse(null); - daggerApp.blockStreamManager().initBlockTrees(state, lastBlockHash); + daggerApp.blockStreamManager().init(state, lastBlockHash); migrationStateChanges = null; } } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java index 542ed10dec1c..c3d7d415d4bd 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java @@ -87,7 +87,7 @@ interface Lifecycle { * @param state the state to use * @param lastBlockHash the hash of the last block */ - void initBlockTrees(@NonNull State state, @Nullable Bytes lastBlockHash); + void init(@NonNull State state, @Nullable Bytes lastBlockHash); /** * Updates the internal state of the block stream manager to reflect the start of a new round. diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 6692936495ba..8ec595aef3d6 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -14,6 +14,7 @@ import static com.hedera.node.app.blocks.impl.streaming.FileBlockItemWriter.cleanUpPendingBlock; import static com.hedera.node.app.blocks.impl.streaming.FileBlockItemWriter.loadContiguousPendingBlocks; import static com.hedera.node.app.blocks.schemas.V0560BlockStreamSchema.BLOCK_STREAM_INFO_STATE_ID; +import static com.hedera.node.app.hapi.utils.CommonUtils.inputOrNullHash; import static com.hedera.node.app.hapi.utils.CommonUtils.noThrowSha384HashOf; import static com.hedera.node.app.hapi.utils.CommonUtils.sha384DigestOrThrow; import static com.hedera.node.app.records.BlockRecordService.EPOCH; @@ -142,11 +143,11 @@ public class BlockStreamManagerImpl implements BlockStreamManager { // The last non-empty (i.e., not skipped) round number that will eventually get a start-of-state hash private Bytes lastBlockHash; private long lastRoundOfPrevBlock; - private Instant blockTimestamp; + // A block's starting timestamp is defined as the consensus timestamp of the round's first transaction + private Instant blockTimestamp; private Instant consensusTimeLastRound; private Timestamp lastUsedTime; private BlockItemWriter writer; - private Instant firstConsensusTimeOfCurrentBlock; // Block merkle subtrees and leaves private IncrementalStreamingHasher previousBlockHashes; @@ -271,7 +272,7 @@ public boolean hasLedgerId() { } @Override - public void initBlockTrees(@NonNull final State state, @Nullable final Bytes lastBlockHash) { + public void init(@NonNull final State state, @Nullable final Bytes lastBlockHash) { final var blockStreamInfo = state.getReadableStates(BlockStreamService.NAME) .getSingleton(BLOCK_STREAM_INFO_STATE_ID) .get(); @@ -313,6 +314,8 @@ public void initBlockTrees(@NonNull final State state, @Nullable final Bytes las .build()) .build(); final var lastStateChanges = BlockItem.newBuilder() + // The final state changes block item for the last block uses blockEndTime, which has to be the last + // state change time .stateChanges(new StateChanges(blockStreamInfo.blockEndTime(), List.of(lastBlockFinalStateChange))) .build(); // Hash the reconstructed (final) state changes block item @@ -332,7 +335,7 @@ public void initBlockTrees(@NonNull final State state, @Nullable final Bytes las blockStreamInfo.outputItemRootHash(), lastBlockFinalStateChangesHash, blockStreamInfo.traceDataRootHash(), - blockStreamInfo.blockStartConsensusTimestamp()) + blockStreamInfo.blockTime()) .blockRootHash()); requireNonNull(calculatedLastBlockHash); this.lastBlockHash = calculatedLastBlockHash; @@ -367,7 +370,7 @@ public void startRound(@NonNull final Round round, @NonNull final State state) { lifecycle.onOpenBlock(state); - resetContainedSubtrees(); + resetSubtrees(); blockNumber = blockStreamInfo.blockNumber() + 1; if (hintsEnabled && !hasCheckedForPendingBlocks) { @@ -389,7 +392,6 @@ public void startRound(@NonNull final Round round, @NonNull final State state) { .blockTimestamp(asTimestamp(blockTimestamp)) .hapiProtoVersion(hapiVersion); worker.addItem(BlockItem.newBuilder().blockHeader(header).build()); - firstConsensusTimeOfCurrentBlock = round.getConsensusTimestamp(); } consensusTimeLastRound = round.getConsensusTimestamp(); } @@ -573,9 +575,11 @@ public boolean endRound(@NonNull final State state, final long roundNum) { asTimestamp(lastIntervalProcessTime), asTimestamp(lastTopLevelTime), consensusHeaderHash, - outputsHash, traceDataHash, - asTimestamp(firstConsensusTimeOfCurrentBlock), + outputsHash, + null, + null, + null, previousBlockHashes.intermediateHashingState(), previousBlockHashes.leafCount()); blockStreamInfoState.put(newBlockStreamInfo); @@ -597,7 +601,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { outputsHash, stateChangesHash, traceDataHash, - newBlockStreamInfo.blockStartConsensusTimestamp()); + newBlockStreamInfo.blockTime()); final var finalBlockRootHash = rootAndSiblingHashes.blockRootHash(); // Create BlockFooter with the three essential hashes: @@ -617,7 +621,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { worker.sync(); // Create a pending block, waiting to be signed - final var blockProofBuilder = BlockProof.newBuilder(); + final var blockProofBuilder = BlockProof.newBuilder().block(blockNumber); pendingBlocks.add(new PendingBlock( blockNumber, null, @@ -761,15 +765,15 @@ private synchronized void finishProofWithSignature( // This must a TssSignedBlockProof since there's a block signature proof = block.proofBuilder().signedBlockProof(latestSignedBlockProof); } else { - // !!!requires(!siblingHashes.isEmpty()) + // This is an indirect proof, thereby requiring a certain number of sibling hashes - // This is an indirect proof (closed with at least one sibling hash) + // (FUTURE) Replace this static indirect proof with the correct three Merkle paths required for a state + // proof to the current block's previous block hash subroot proof = block.proofBuilder() .blockStateProof(StateProof.newBuilder() .paths(MerklePath.newBuilder().build()) .signedBlockProof(latestSignedBlockProof) .build()) - // TODO: Is this right?? Does verification require sibling _block_ hashes? .siblingHashes( siblingHashes.stream().flatMap(List::stream).toList()); @@ -1125,7 +1129,7 @@ private BlockItem flushChangesFromListener(@NonNull final BoundaryStateChangeLis * Resets the subtree hashers for branches 4-8 to empty states. Since these subtrees only contain data specific to * the current block, they need to be reset whenever a new block starts. */ - private void resetContainedSubtrees() { + private void resetSubtrees() { // Branch 4 consensusHeaderHasher = new ConcurrentStreamingTreeHasher(executor, hashCombineBatchSize); // Branch 5 @@ -1156,26 +1160,15 @@ private static RootAndSiblingHashes combine( @Nullable final Bytes maybeOutputsHash, @Nullable final Bytes maybeStateChangesHash, @Nullable final Bytes maybeTraceDataHash, - final Timestamp firstConsensusTimeOfCurrentBlock) { - final var prevBlockHash = - (maybePrevBlockHash != null && maybePrevBlockHash.length() > 0) ? maybePrevBlockHash : NULL_HASH; - final var prevBlockRootsHash = (maybePrevBlockRootsHash != null && maybePrevBlockRootsHash.length() > 0) - ? maybePrevBlockRootsHash - : NULL_HASH; - final var startingStateHash = (maybeStartingStateHash != null && maybeStartingStateHash.length() > 0) - ? maybeStartingStateHash - : NULL_HASH; - final var consensusHeaderHash = (maybeConsensusHeaderHash != null && maybeConsensusHeaderHash.length() > 0) - ? maybeConsensusHeaderHash - : NULL_HASH; - final var inputsHash = (maybeInputsHash != null && maybeInputsHash.length() > 0) ? maybeInputsHash : NULL_HASH; - final var outputsHash = - (maybeOutputsHash != null && maybeOutputsHash.length() > 0) ? maybeOutputsHash : NULL_HASH; - final var stateChangesHash = (maybeStateChangesHash != null && maybeStateChangesHash.length() > 0) - ? maybeStateChangesHash - : NULL_HASH; - final var traceDataHash = - (maybeTraceDataHash != null && maybeTraceDataHash.length() > 0) ? maybeTraceDataHash : NULL_HASH; + @NonNull final Timestamp firstConsensusTimeOfCurrentBlock) { + final var prevBlockHash = inputOrNullHash(maybePrevBlockHash); + final var prevBlockRootsHash = inputOrNullHash(maybePrevBlockRootsHash); + final var startingStateHash = inputOrNullHash(maybeStartingStateHash); + final var consensusHeaderHash = inputOrNullHash(maybeConsensusHeaderHash); + final var inputsHash = inputOrNullHash(maybeInputsHash); + final var outputsHash = inputOrNullHash(maybeOutputsHash); + final var stateChangesHash = inputOrNullHash(maybeStateChangesHash); + final var traceDataHash = inputOrNullHash(maybeTraceDataHash); // Compute depth four hashes final var depth4Node1 = BlockImplUtils.combine(prevBlockHash, prevBlockRootsHash); From 20e0405d81a3b0449c6fb72c8be152fe23659e5f Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sat, 1 Nov 2025 01:57:30 -0600 Subject: [PATCH 40/63] Some updates to state changes validator Signed-off-by: Matt Hess --- .../block/StateChangesValidator.java | 129 ++++++++++++++---- 1 file changed, 100 insertions(+), 29 deletions(-) diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java index af2270abc8b0..33fd67f62e74 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java @@ -12,6 +12,8 @@ import static com.hedera.hapi.util.HapiUtils.asInstant; import static com.hedera.node.app.blocks.impl.BlockImplUtils.combine; import static com.hedera.node.app.blocks.impl.BlockStreamManagerImpl.NULL_HASH; +import static com.hedera.node.app.hapi.utils.CommonUtils.inputOrNullHash; +import static com.hedera.node.app.hapi.utils.CommonUtils.noThrowSha384HashOf; import static com.hedera.node.app.hapi.utils.CommonUtils.sha384DigestOrThrow; import static com.hedera.node.app.hapi.utils.blocks.BlockStreamUtils.stateNameOf; import static com.hedera.node.app.hints.HintsService.maybeWeightsFrom; @@ -35,8 +37,11 @@ import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.hapi.block.stream.BlockProof; +import com.hedera.hapi.block.stream.output.BlockFooter; +import com.hedera.hapi.block.stream.output.StateChange; import com.hedera.hapi.block.stream.output.StateChanges; import com.hedera.hapi.block.stream.output.StateIdentifier; +import com.hedera.hapi.node.base.Timestamp; import com.hedera.hapi.node.state.entity.EntityCounts; import com.hedera.hapi.node.state.hints.HintsConstruction; import com.hedera.hapi.node.state.hints.PreprocessedKeys; @@ -49,8 +54,11 @@ import com.hedera.node.app.ServicesMain; import com.hedera.node.app.blocks.BlockStreamManager; import com.hedera.node.app.blocks.StreamingTreeHasher; +import com.hedera.node.app.blocks.impl.BlockImplUtils; +import com.hedera.node.app.blocks.impl.IncrementalStreamingHasher; import com.hedera.node.app.blocks.impl.NaiveStreamingTreeHasher; import com.hedera.node.app.config.BootstrapConfigProviderImpl; +import com.hedera.node.app.hapi.utils.CommonUtils; import com.hedera.node.app.hapi.utils.blocks.BlockStreamAccess; import com.hedera.node.app.hapi.utils.blocks.BlockStreamUtils; import com.hedera.node.app.hints.HintsLibrary; @@ -91,6 +99,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.SortedMap; @@ -343,6 +352,8 @@ public void validateBlocks(@NonNull final List blocks) { .filter(HintsConstruction::hasHintsScheme) .forEach(c -> preprocessedKeys.put( c.constructionId(), c.hintsSchemeOrThrow().preprocessedKeysOrThrow())); + final IncrementalStreamingHasher incrementalBlockHashes = + new IncrementalStreamingHasher(CommonUtils.sha384DigestOrThrow(), List.of(), 0); for (int i = 0; i < n; i++) { final var block = blocks.get(i); final var shouldVerifyProof = @@ -361,7 +372,15 @@ public void validateBlocks(@NonNull final List blocks) { long firstBlockRound = -1; long eventNodeId = -1; + Timestamp firstConsensusTimestamp = null; for (final var item : block.items()) { + if (firstConsensusTimestamp == null && item.hasBlockHeader()) { + firstConsensusTimestamp = item.blockHeaderOrThrow().blockTimestamp(); + assertTrue( + firstConsensusTimestamp != null + && !Objects.equals(firstConsensusTimestamp, Timestamp.DEFAULT), + "Block header timestamp is unset"); + } if (firstBlockRound == -1 && item.hasRoundHeader()) { firstBlockRound = item.roundHeaderOrThrow().roundNumber(); } @@ -404,34 +423,55 @@ public void validateBlocks(@NonNull final List blocks) { } } if (i <= lastVerifiableIndex) { + final var footer = block.items().stream() + .filter(BlockItem::hasBlockFooter) + .map(BlockItem::blockFooterOrThrow) + .findFirst() + .orElseThrow(); final var lastBlockItem = block.items().getLast(); assertTrue(lastBlockItem.hasBlockProof()); final var blockProof = lastBlockItem.blockProofOrThrow(); - // TODO: get previous block root hash - // assertEquals( - // previousBlockHash, - // blockProof.previousBlockRootHash(), - // "Previous block hash mismatch for block " + blockProof.block()); + assertEquals( + previousBlockHash, + footer.previousBlockRootHash(), + "Previous block hash mismatch for block " + blockProof.block()); if (shouldVerifyProof) { + final var lastStateChange = lastStateChanges.stateChanges().getLast(); + assertTrue( + lastStateChange.hasSingletonUpdate(), + "Final state change " + lastStateChange + " does not match expected singleton update type"); + assertTrue( + lastStateChange.singletonUpdateOrThrow().hasBlockStreamInfoValue(), + "Final state change " + lastStateChange + + " does not match final block BlockStreamInfo update type"); + + final var penultimateStateChangesHash = + stateChangesHasher.rootHash().join(); + final var hashedChangeBytes = noThrowSha384HashOf(StateChange.PROTOBUF.toBytes(lastStateChange)); + + // Combine the penultimate state change leaf with the final state change leaf + final var finalStateChangesHash = + BlockImplUtils.combine(penultimateStateChangesHash, hashedChangeBytes); + final var expectedBlockHash = computeBlockHash( - startOfStateHash, + firstConsensusTimestamp, previousBlockHash, + incrementalBlockHashes, + startOfStateHash, inputTreeHasher, outputTreeHasher, consensusHeaderHasher, - stateChangesHasher, + finalStateChangesHash, traceDataHasher); blockNumbers.put( expectedBlockHash, block.items().getFirst().blockHeaderOrThrow().number()); - validateBlockProof(i, firstBlockRound, blockProof, expectedBlockHash, startOfStateHash); + validateBlockProof(i, firstBlockRound, footer, blockProof, expectedBlockHash, startOfStateHash); + incrementalBlockHashes.addLeaf(expectedBlockHash.toByteArray()); previousBlockHash = expectedBlockHash; } else { - // TODO: get previous block root hash - // previousBlockHash = requireNonNull( - // blocks.get(i + 1).items().getLast().blockProof()) - // .previousBlockRootHash(); + previousBlockHash = footer.previousBlockRootHash(); } } } @@ -544,36 +584,67 @@ private void hashSubTrees( } private Bytes computeBlockHash( - final Bytes startOfBlockStateHash, - final Bytes previousBlockHash, + final Timestamp blockTimestamp, + final Bytes maybePreviousBlockHash, + final IncrementalStreamingHasher prevBlockRootsHasher, + final Bytes maybeStartOfBlockStateHash, final StreamingTreeHasher inputTreeHasher, final StreamingTreeHasher outputTreeHasher, final StreamingTreeHasher consensusHeaderHasher, - final StreamingTreeHasher stateChangesHasher, + final Bytes maybeFinalStateChangesHash, final StreamingTreeHasher traceDataHasher) { - final var inputTreeHash = inputTreeHasher.rootHash().join(); - final var outputTreeHash = outputTreeHasher.rootHash().join(); - final var consensusHeaderHash = consensusHeaderHasher.rootHash().join(); - final var stateChangesHash = stateChangesHasher.rootHash().join(); - final var traceDataHash = traceDataHasher.rootHash().join(); - - final var leftParent = - combine(combine(previousBlockHash, startOfBlockStateHash), combine(consensusHeaderHash, inputTreeHash)); - final var rightParent = combine(combine(outputTreeHash, stateChangesHash), combine(traceDataHash, NULL_HASH)); - return combine(leftParent, rightParent); + final var previousBlockHash = inputOrNullHash(maybePreviousBlockHash); + final var prevBlocksRootHash = inputOrNullHash(Bytes.wrap(prevBlockRootsHasher.computeRootHash())); + final var startOfBlockStateHash = inputOrNullHash(maybeStartOfBlockStateHash); + final var consensusHeaderHash = + inputOrNullHash(consensusHeaderHasher.rootHash().join()); + final var inputTreeHash = inputOrNullHash(inputTreeHasher.rootHash().join()); + final var outputTreeHash = inputOrNullHash(outputTreeHasher.rootHash().join()); + final var finalStateChangesHash = inputOrNullHash(maybeFinalStateChangesHash); + final var traceDataHash = inputOrNullHash(traceDataHasher.rootHash().join()); + + // Compute depth four hashes + final var depth4Node1 = BlockImplUtils.combine(previousBlockHash, prevBlocksRootHash); + final var depth4Node2 = BlockImplUtils.combine(startOfBlockStateHash, consensusHeaderHash); + final var depth4Node3 = BlockImplUtils.combine(inputTreeHash, outputTreeHash); + final var depth4Node4 = BlockImplUtils.combine(finalStateChangesHash, traceDataHash); + + final var combinedNulls = BlockImplUtils.combine(NULL_HASH, NULL_HASH); + // Nodes 5-8 for depth four are all combined null hashes, but enumerated for clarity + final var depth4Node5 = combinedNulls; + final var depth4Node6 = combinedNulls; + final var depth4Node7 = combinedNulls; + final var depth4Node8 = combinedNulls; + + // Compute depth three hashes + final var depth3Node1 = BlockImplUtils.combine(depth4Node1, depth4Node2); + final var depth3Node2 = BlockImplUtils.combine(depth4Node3, depth4Node4); + final var depth3Node3 = BlockImplUtils.combine(depth4Node5, depth4Node6); + final var depth3Node4 = BlockImplUtils.combine(depth4Node7, depth4Node8); + + // Compute depth two hashes + final var depth2Node1 = BlockImplUtils.combine(depth3Node1, depth3Node2); + final var depth2Node2 = BlockImplUtils.combine(depth3Node3, depth3Node4); + + // Compute depth one hashes + final var timestamp = Timestamp.PROTOBUF.toBytes(blockTimestamp); + final var depth1Node0 = noThrowSha384HashOf(timestamp); + final var depth1Node1 = BlockImplUtils.combine(depth2Node1, depth2Node2); + + // Compute the block's root hash + return BlockImplUtils.combine(depth1Node0, depth1Node1); } private void validateBlockProof( final long number, final long firstRound, + @NonNull final BlockFooter footer, @NonNull final BlockProof proof, @NonNull final Bytes blockHash, @NonNull final Bytes startOfStateHash) { assertEquals(number, proof.block()); - // TODO: get start of block state root hash - // assertEquals( - // proof.startOfBlockStateRootHash(), startOfStateHash, "Wrong start of state hash for block #" + - // number); + assertEquals( + footer.startOfBlockStateRootHash(), startOfStateHash, "Wrong start of state hash for block #" + number); var provenHash = blockHash; final var siblingHashes = proof.siblingHashes(); if (!siblingHashes.isEmpty()) { From 165d857966adea5a5801ede5e85f3a4e3ed40761 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sat, 1 Nov 2025 02:09:40 -0600 Subject: [PATCH 41/63] Cleanup Signed-off-by: Matt Hess --- .../state/blockstream/block_stream_info.proto | 13 +++++-------- .../app/blocks/impl/BlockStreamManagerImpl.java | 13 +++++-------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto index 0e211e18f0fb..8362b9fbcbe2 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto @@ -167,19 +167,16 @@ message BlockStreamInfo { */ bytes trace_data_root_hash = 17; - /** - * The time stamp at which the first transaction was handled in - * this block. - */ - proto.Timestamp block_start_consensus_timestamp = 18; - /** * The intermediate hashes needed for branch 2 in the block merkle * tree structure. These hashes SHALL include all block root hashes * needed to construct branch 2's final state at the end of the * previous block */ - repeated bytes intermediate_previous_block_root_hashes = 19; + repeated bytes intermediate_previous_block_root_hashes = 18; - uint64 intermediate_block_roots_leaf_count = 20; + /** + * The number of leaves in the intermediate block roots subtree. + */ + uint64 intermediate_block_roots_leaf_count = 19; } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 8ec595aef3d6..10532db6223c 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -143,8 +143,8 @@ public class BlockStreamManagerImpl implements BlockStreamManager { // The last non-empty (i.e., not skipped) round number that will eventually get a start-of-state hash private Bytes lastBlockHash; private long lastRoundOfPrevBlock; - // A block's starting timestamp is defined as the consensus timestamp of the round's first transaction - private Instant blockTimestamp; + // A block's starting timestamp is defined as the consensus timestamp of the round's first transaction + private Instant blockTimestamp; private Instant consensusTimeLastRound; private Timestamp lastUsedTime; private BlockItemWriter writer; @@ -575,11 +575,8 @@ public boolean endRound(@NonNull final State state, final long roundNum) { asTimestamp(lastIntervalProcessTime), asTimestamp(lastTopLevelTime), consensusHeaderHash, - traceDataHash, outputsHash, - null, - null, - null, + traceDataHash, previousBlockHashes.intermediateHashingState(), previousBlockHashes.leafCount()); blockStreamInfoState.put(newBlockStreamInfo); @@ -767,8 +764,8 @@ private synchronized void finishProofWithSignature( } else { // This is an indirect proof, thereby requiring a certain number of sibling hashes - // (FUTURE) Replace this static indirect proof with the correct three Merkle paths required for a state - // proof to the current block's previous block hash subroot + // (FUTURE) Replace this static indirect proof with the correct three Merkle paths required for a state + // proof to the current block's previous block hash subroot proof = block.proofBuilder() .blockStateProof(StateProof.newBuilder() .paths(MerklePath.newBuilder().build()) From 38244c540707677845b458a97cff780d4a1622c2 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sat, 1 Nov 2025 15:49:57 -0600 Subject: [PATCH 42/63] Fix a couple state changes validator issues Signed-off-by: Matt Hess --- .../block/StateChangesValidator.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java index 33fd67f62e74..0b00aeb01275 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java @@ -10,6 +10,7 @@ import static com.hedera.hapi.block.stream.output.StateIdentifier.STATE_ID_ROSTER_STATE; import static com.hedera.hapi.node.base.HederaFunctionality.HINTS_PARTIAL_SIGNATURE; import static com.hedera.hapi.util.HapiUtils.asInstant; +import static com.hedera.hapi.util.HapiUtils.asTimestamp; import static com.hedera.node.app.blocks.impl.BlockImplUtils.combine; import static com.hedera.node.app.blocks.impl.BlockStreamManagerImpl.NULL_HASH; import static com.hedera.node.app.hapi.utils.CommonUtils.inputOrNullHash; @@ -29,6 +30,7 @@ import static com.hedera.services.bdd.junit.support.validators.block.RootHashUtils.extractRootMnemonic; import static com.hedera.services.bdd.spec.TargetNetworkType.SUBPROCESS_NETWORK; import static com.swirlds.platform.system.InitTrigger.GENESIS; +import static java.time.Instant.EPOCH; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toMap; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -38,7 +40,6 @@ import com.hedera.hapi.block.stream.BlockItem; import com.hedera.hapi.block.stream.BlockProof; import com.hedera.hapi.block.stream.output.BlockFooter; -import com.hedera.hapi.block.stream.output.StateChange; import com.hedera.hapi.block.stream.output.StateChanges; import com.hedera.hapi.block.stream.output.StateIdentifier; import com.hedera.hapi.node.base.Timestamp; @@ -354,6 +355,7 @@ public void validateBlocks(@NonNull final List blocks) { c.constructionId(), c.hintsSchemeOrThrow().preprocessedKeysOrThrow())); final IncrementalStreamingHasher incrementalBlockHashes = new IncrementalStreamingHasher(CommonUtils.sha384DigestOrThrow(), List.of(), 0); + incrementalBlockHashes.addLeaf(BlockStreamManager.ZERO_BLOCK_HASH.toByteArray()); for (int i = 0; i < n; i++) { final var block = blocks.get(i); final var shouldVerifyProof = @@ -373,6 +375,7 @@ public void validateBlocks(@NonNull final List blocks) { long firstBlockRound = -1; long eventNodeId = -1; Timestamp firstConsensusTimestamp = null; + Timestamp lastConsensusTimestamp = asTimestamp(EPOCH); for (final var item : block.items()) { if (firstConsensusTimestamp == null && item.hasBlockHeader()) { firstConsensusTimestamp = item.blockHeaderOrThrow().blockTimestamp(); @@ -384,6 +387,15 @@ public void validateBlocks(@NonNull final List blocks) { if (firstBlockRound == -1 && item.hasRoundHeader()) { firstBlockRound = item.roundHeaderOrThrow().roundNumber(); } + if (item.hasStateChanges()) { + final var thisItemTimestamp = Optional.ofNullable(item.stateChanges()) + .map(StateChanges::consensusTimestamp) + .orElse(null); + if (thisItemTimestamp != null + && asInstant(lastConsensusTimestamp).isBefore(asInstant(thisItemTimestamp))) { + lastConsensusTimestamp = thisItemTimestamp; + } + } if (shouldVerifyProof) { hashSubTrees( item, @@ -446,13 +458,9 @@ public void validateBlocks(@NonNull final List blocks) { "Final state change " + lastStateChange + " does not match final block BlockStreamInfo update type"); - final var penultimateStateChangesHash = - stateChangesHasher.rootHash().join(); - final var hashedChangeBytes = noThrowSha384HashOf(StateChange.PROTOBUF.toBytes(lastStateChange)); - - // Combine the penultimate state change leaf with the final state change leaf + // The state changes hasher already incorporated the last state change, so compute its root hash final var finalStateChangesHash = - BlockImplUtils.combine(penultimateStateChangesHash, hashedChangeBytes); + stateChangesHasher.rootHash().join(); final var expectedBlockHash = computeBlockHash( firstConsensusTimestamp, @@ -468,11 +476,12 @@ public void validateBlocks(@NonNull final List blocks) { expectedBlockHash, block.items().getFirst().blockHeaderOrThrow().number()); validateBlockProof(i, firstBlockRound, footer, blockProof, expectedBlockHash, startOfStateHash); - incrementalBlockHashes.addLeaf(expectedBlockHash.toByteArray()); previousBlockHash = expectedBlockHash; } else { previousBlockHash = footer.previousBlockRootHash(); } + + incrementalBlockHashes.addLeaf(previousBlockHash.toByteArray()); } } logger.info("Summary of changes by service:\n{}", stateChangesSummary); From 6ad0fae1af41ddc2685cd13291bcafa166d45f9b Mon Sep 17 00:00:00 2001 From: Neeharika-Sompalli Date: Sun, 2 Nov 2025 15:37:20 -0600 Subject: [PATCH 43/63] wip Signed-off-by: Neeharika-Sompalli --- .../state/blockstream/block_stream_info.proto | 10 ++++- .../blocks/impl/BlockStreamManagerImpl.java | 7 ++- .../block/StateChangesValidator.java | 43 +++++++++---------- .../bdd/suites/crypto/CryptoCreateSuite.java | 2 + .../hip1195/Hip1195StreamParityTest.java | 10 ++--- 5 files changed, 40 insertions(+), 32 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto index 8362b9fbcbe2..d8ad37f44bee 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto @@ -167,16 +167,22 @@ message BlockStreamInfo { */ bytes trace_data_root_hash = 17; + /** + * The time stamp at which the first transaction was handled in + * this block. + */ + proto.Timestamp block_start_consensus_timestamp = 18; + /** * The intermediate hashes needed for branch 2 in the block merkle * tree structure. These hashes SHALL include all block root hashes * needed to construct branch 2's final state at the end of the * previous block */ - repeated bytes intermediate_previous_block_root_hashes = 18; + repeated bytes intermediate_previous_block_root_hashes = 19; /** * The number of leaves in the intermediate block roots subtree. */ - uint64 intermediate_block_roots_leaf_count = 19; + uint64 intermediate_block_roots_leaf_count = 20; } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 10532db6223c..02b7d2af622d 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -148,6 +148,7 @@ public class BlockStreamManagerImpl implements BlockStreamManager { private Instant consensusTimeLastRound; private Timestamp lastUsedTime; private BlockItemWriter writer; + private Instant firstConsensusTimeOfCurrentBlock; // Block merkle subtrees and leaves private IncrementalStreamingHasher previousBlockHashes; @@ -335,7 +336,7 @@ public void init(@NonNull final State state, @Nullable final Bytes lastBlockHash blockStreamInfo.outputItemRootHash(), lastBlockFinalStateChangesHash, blockStreamInfo.traceDataRootHash(), - blockStreamInfo.blockTime()) + blockStreamInfo.blockStartConsensusTimestamp()) .blockRootHash()); requireNonNull(calculatedLastBlockHash); this.lastBlockHash = calculatedLastBlockHash; @@ -392,6 +393,7 @@ public void startRound(@NonNull final Round round, @NonNull final State state) { .blockTimestamp(asTimestamp(blockTimestamp)) .hapiProtoVersion(hapiVersion); worker.addItem(BlockItem.newBuilder().blockHeader(header).build()); + firstConsensusTimeOfCurrentBlock = round.getConsensusTimestamp(); } consensusTimeLastRound = round.getConsensusTimestamp(); } @@ -577,6 +579,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { consensusHeaderHash, outputsHash, traceDataHash, + asTimestamp(firstConsensusTimeOfCurrentBlock), previousBlockHashes.intermediateHashingState(), previousBlockHashes.leafCount()); blockStreamInfoState.put(newBlockStreamInfo); @@ -598,7 +601,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { outputsHash, stateChangesHash, traceDataHash, - newBlockStreamInfo.blockTime()); + newBlockStreamInfo.blockStartConsensusTimestamp()); final var finalBlockRootHash = rootAndSiblingHashes.blockRootHash(); // Create BlockFooter with the three essential hashes: diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java index 0b00aeb01275..12833a3971f6 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java @@ -10,7 +10,6 @@ import static com.hedera.hapi.block.stream.output.StateIdentifier.STATE_ID_ROSTER_STATE; import static com.hedera.hapi.node.base.HederaFunctionality.HINTS_PARTIAL_SIGNATURE; import static com.hedera.hapi.util.HapiUtils.asInstant; -import static com.hedera.hapi.util.HapiUtils.asTimestamp; import static com.hedera.node.app.blocks.impl.BlockImplUtils.combine; import static com.hedera.node.app.blocks.impl.BlockStreamManagerImpl.NULL_HASH; import static com.hedera.node.app.hapi.utils.CommonUtils.inputOrNullHash; @@ -30,7 +29,6 @@ import static com.hedera.services.bdd.junit.support.validators.block.RootHashUtils.extractRootMnemonic; import static com.hedera.services.bdd.spec.TargetNetworkType.SUBPROCESS_NETWORK; import static com.swirlds.platform.system.InitTrigger.GENESIS; -import static java.time.Instant.EPOCH; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toMap; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -210,13 +208,13 @@ public static void main(String[] args) { final long realm = 12; final var validator = new StateChangesValidator( Bytes.fromHex( - "525279ce448629033053af7fd64e1439f415c0acb5ad6819b73363807122847b2d68ded6d47db36b59920474093f0651"), + "bd2ee10d715acd195587977ee3f259e68cc84a55cc593a9789b36ef91aee3548434dec3149e4bf5ddecc020e0fbf1193"), node0Dir.resolve("output/swirlds.log"), node0Dir.resolve("data/config/application.properties"), node0Dir.resolve("data/config"), 16, - HintsEnabled.YES, - HistoryEnabled.YES, + HintsEnabled.NO, + HistoryEnabled.NO, hintsThresholdDenominator, shard, realm); @@ -375,7 +373,6 @@ public void validateBlocks(@NonNull final List blocks) { long firstBlockRound = -1; long eventNodeId = -1; Timestamp firstConsensusTimestamp = null; - Timestamp lastConsensusTimestamp = asTimestamp(EPOCH); for (final var item : block.items()) { if (firstConsensusTimestamp == null && item.hasBlockHeader()) { firstConsensusTimestamp = item.blockHeaderOrThrow().blockTimestamp(); @@ -387,15 +384,6 @@ public void validateBlocks(@NonNull final List blocks) { if (firstBlockRound == -1 && item.hasRoundHeader()) { firstBlockRound = item.roundHeaderOrThrow().roundNumber(); } - if (item.hasStateChanges()) { - final var thisItemTimestamp = Optional.ofNullable(item.stateChanges()) - .map(StateChanges::consensusTimestamp) - .orElse(null); - if (thisItemTimestamp != null - && asInstant(lastConsensusTimestamp).isBefore(asInstant(thisItemTimestamp))) { - lastConsensusTimestamp = thisItemTimestamp; - } - } if (shouldVerifyProof) { hashSubTrees( item, @@ -435,17 +423,14 @@ && asInstant(lastConsensusTimestamp).isBefore(asInstant(thisItemTimestamp))) { } } if (i <= lastVerifiableIndex) { - final var footer = block.items().stream() - .filter(BlockItem::hasBlockFooter) - .map(BlockItem::blockFooterOrThrow) - .findFirst() - .orElseThrow(); + final var footer = block.items().get(block.items().size() - 2); + assertTrue(footer.hasBlockFooter()); final var lastBlockItem = block.items().getLast(); assertTrue(lastBlockItem.hasBlockProof()); final var blockProof = lastBlockItem.blockProofOrThrow(); assertEquals( previousBlockHash, - footer.previousBlockRootHash(), + footer.blockFooterOrThrow().previousBlockRootHash(), "Previous block hash mismatch for block " + blockProof.block()); if (shouldVerifyProof) { @@ -475,10 +460,22 @@ && asInstant(lastConsensusTimestamp).isBefore(asInstant(thisItemTimestamp))) { blockNumbers.put( expectedBlockHash, block.items().getFirst().blockHeaderOrThrow().number()); - validateBlockProof(i, firstBlockRound, footer, blockProof, expectedBlockHash, startOfStateHash); + validateBlockProof( + i, + firstBlockRound, + footer.blockFooterOrThrow(), + blockProof, + expectedBlockHash, + startOfStateHash); previousBlockHash = expectedBlockHash; } else { - previousBlockHash = footer.previousBlockRootHash(); + final var nextBlock = blocks.get(i + 1); + final var nextBlockFooterIndex = nextBlock.items().size() - 2; + previousBlockHash = nextBlock + .items() + .get(nextBlockFooterIndex) + .blockFooterOrThrow() + .previousBlockRootHash(); } incrementalBlockHashes.addLeaf(previousBlockHash.toByteArray()); diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/crypto/CryptoCreateSuite.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/crypto/CryptoCreateSuite.java index f57de08c8c93..a08f6da7f119 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/crypto/CryptoCreateSuite.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/crypto/CryptoCreateSuite.java @@ -2,6 +2,7 @@ package com.hedera.services.bdd.suites.crypto; import static com.hedera.node.app.hapi.utils.EthSigsUtils.recoverAddressFromPubKey; +import static com.hedera.services.bdd.junit.TestTags.ADHOC; import static com.hedera.services.bdd.junit.TestTags.CRYPTO; import static com.hedera.services.bdd.junit.TestTags.MATS; import static com.hedera.services.bdd.spec.HapiSpec.hapiTest; @@ -105,6 +106,7 @@ final Stream idVariantsTreatedAsExpected() { } @HapiTest + @Tag(ADHOC) public Stream cantCreateTwoAccountsWithSameAlias() { final String ecKey = "ecKey"; final String key1 = "key1"; diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/hip1195/Hip1195StreamParityTest.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/hip1195/Hip1195StreamParityTest.java index b73b8c152377..378c6406dd8f 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/hip1195/Hip1195StreamParityTest.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/hip1195/Hip1195StreamParityTest.java @@ -61,7 +61,7 @@ @SuppressWarnings({"rawtypes", "unchecked"}) @Tag(ADHOC) public class Hip1195StreamParityTest { - public static final String HOOK_CONTRACT = "0.0.365"; + public static final String HOOK_CONTRACT_NUM = "365"; private static final TupleType SET_AND_PASS_ARGS = TupleType.parse("(uint32,address)"); @@ -260,16 +260,16 @@ final Stream hookExecutionsWithAutoCreations() { recordWith().status(SUCCESS).memo(AUTO_MEMO), recordWith() .status(SUCCESS) - .contractCallResult(resultWith().contract(HOOK_CONTRACT)), + .contractCallResult(resultWith().contract(HOOK_CONTRACT_NUM)), recordWith() .status(SUCCESS) - .contractCallResult(resultWith().contract(HOOK_CONTRACT)), + .contractCallResult(resultWith().contract(HOOK_CONTRACT_NUM)), recordWith() .status(SUCCESS) - .contractCallResult(resultWith().contract(HOOK_CONTRACT)), + .contractCallResult(resultWith().contract(HOOK_CONTRACT_NUM)), recordWith() .status(SUCCESS) - .contractCallResult(resultWith().contract(HOOK_CONTRACT))) + .contractCallResult(resultWith().contract(HOOK_CONTRACT_NUM))) .logged(), getAliasedAccountInfo("alias").has(accountWith().balance(10L)).hasToken(relationshipWith("tokenA"))); } From a7edd85cb4c58455577bf19476e73501e75fa482 Mon Sep 17 00:00:00 2001 From: Neeharika-Sompalli Date: Sun, 2 Nov 2025 16:11:55 -0600 Subject: [PATCH 44/63] fix translator Signed-off-by: Neeharika-Sompalli --- .../node/app/blocks/impl/BlockStreamBuilder.java | 16 ++++++++++++---- .../support/translators/BaseTranslator.java | 6 +++--- .../src/main/resources/spec-default.properties | 1 + 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java index 1f860a4d2e8e..0c1ad158e7ab 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamBuilder.java @@ -547,7 +547,15 @@ private T toView(@NonNull final BlockItemsTranslator translator, @NonNull fi } List logs = null; for (final var item : blockItems.subList(j, n)) { - // TODO: new trace data implementation + if (item.hasTraceData()) { + final var traceData = item.traceDataOrThrow(); + if (traceData.hasEvmTraceData()) { + if (logs == null) { + logs = new ArrayList<>(); + } + logs.addAll(traceData.evmTraceDataOrThrow().logs()); + } + } } return (T) switch (view) { @@ -708,7 +716,7 @@ public Output build(final boolean topLevel, @Nullable final List gr builder.logs(logs); } blockItems.add(BlockItem.newBuilder() - // TODO: re-add trace data + .traceData(TraceData.newBuilder().evmTraceData(builder)) .build()); } @@ -720,7 +728,7 @@ public Output build(final boolean topLevel, @Nullable final List gr .automaticTokenAssociations( automaticTokenAssociations.getLast().accountId()); blockItems.add(BlockItem.newBuilder() - // TODO: re-add trace data + .traceData(TraceData.newBuilder().autoAssociateTraceData(builder)) .build()); } // message submit trace data @@ -729,7 +737,7 @@ public Output build(final boolean topLevel, @Nullable final List gr .sequenceNumber(sequenceNumber) .runningHash(runningHash); blockItems.add(BlockItem.newBuilder() - // TODO: re-add trace data + .traceData(TraceData.newBuilder().submitMessageTraceData(builder)) .build()); } } diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/translators/BaseTranslator.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/translators/BaseTranslator.java index 98ac0e3fa4d9..939f29194df5 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/translators/BaseTranslator.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/translators/BaseTranslator.java @@ -1054,7 +1054,7 @@ private static boolean isContractOp(@NonNull final BlockTransactionParts parts) private static Account findContractOrThrow( @NonNull final ContractID contractId, @NonNull final List stateChanges) { - return stateChanges.stream() + final var temp = stateChanges.stream() .filter(change -> change.stateId() == STATE_ID_ACCOUNTS.protoOrdinal()) .filter(StateChange::hasMapUpdate) .map(StateChange::mapUpdateOrThrow) @@ -1068,8 +1068,8 @@ private static Account findContractOrThrow( && contractId.realmNum() == accountId.realmNum() && contractId.contractNumOrThrow().longValue() == accountId.accountNumOrThrow(); }) - .findFirst() - .orElseThrow(); + .findFirst(); + return temp.orElseThrow(); } private static Optional findAccount( diff --git a/hedera-node/test-clients/src/main/resources/spec-default.properties b/hedera-node/test-clients/src/main/resources/spec-default.properties index de4add6f32c6..235d88f78828 100644 --- a/hedera-node/test-clients/src/main/resources/spec-default.properties +++ b/hedera-node/test-clients/src/main/resources/spec-default.properties @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 + address.book.id=101 address.book.name=ADDRESS_BOOK address.book.controlAccount.id=55 From 59f91e4cf4416e723f03bf108dc9acfd101a3841 Mon Sep 17 00:00:00 2001 From: Neeharika-Sompalli Date: Sun, 2 Nov 2025 16:30:10 -0600 Subject: [PATCH 45/63] validation passes on adHoc Signed-off-by: Neeharika-Sompalli --- .../blocks/impl/IncrementalHasherStorage.java | 64 --------- hedera-node/test-clients/build.gradle.kts | 2 +- .../block/StateChangesValidator.java | 130 +++++++++--------- 3 files changed, 63 insertions(+), 133 deletions(-) delete mode 100644 hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java deleted file mode 100644 index 054b82f89911..000000000000 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalHasherStorage.java +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -package com.hedera.node.app.blocks.impl; - -import com.hedera.hapi.node.state.blockstream.StreamingTreeSnapshot; -import com.hedera.hapi.node.state.blockstream.SubMerkleTree; -import com.hedera.pbj.runtime.io.buffer.Bytes; -import edu.umd.cs.findbugs.annotations.NonNull; -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; - -/** - * Storage and reconstruction utility for {@link IncrementalStreamingHasher}. - */ -class IncrementalHasherStorage { - private static final int BUFFER_SIZE = 4 * 1024; // 4KB - - static StreamingTreeSnapshot readStreamingSnapshot(@NonNull final String filepath) { - // todo - return null; - } - - /** - * todo - * @param basepath - * @param hashingStates - * @param roundNum - */ - static void writeStreamingSnapshots( - @NonNull final String basepath, Map> hashingStates, final long roundNum) { - Path created; - try { - created = Files.createDirectory(Path.of(basepath).resolve(String.valueOf(roundNum))); - } catch (IOException e) { - throw new RuntimeException(e); - } - - // write each tree - hashingStates.forEach((type, hasher) -> { - final var snapshot = - StreamingTreeSnapshot.newBuilder().type(type).nodes(hasher).build(); - final Path treePath; - try { - treePath = Files.createFile(created.resolve(filenameFor(type))); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - - try (BufferedOutputStream out = new BufferedOutputStream(Files.newOutputStream(treePath), BUFFER_SIZE)) { - out.write(StreamingTreeSnapshot.PROTOBUF.toBytes(snapshot).toByteArray()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - } - - static String filenameFor(@NonNull final SubMerkleTree type) { - return type.protoName(); - } -} diff --git a/hedera-node/test-clients/build.gradle.kts b/hedera-node/test-clients/build.gradle.kts index a6b02c28852a..c07c5157226b 100644 --- a/hedera-node/test-clients/build.gradle.kts +++ b/hedera-node/test-clients/build.gradle.kts @@ -125,7 +125,7 @@ val prCheckPropOverrides = buildMap { put( "hapiTestAdhoc", - "tss.hintsEnabled=false,tss.forceHandoffs=false,tss.initialCrsParties=16,blockStream.blockPeriod=2s", + "tss.hintsEnabled=true,tss.forceHandoffs=false,tss.initialCrsParties=16,blockStream.blockPeriod=2s", ) put( "hapiTestCrypto", diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java index 12833a3971f6..aaed04019b18 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java @@ -19,6 +19,7 @@ import static com.hedera.node.app.hints.HintsService.maybeWeightsFrom; import static com.hedera.node.app.history.impl.ProofControllerImpl.EMPTY_PUBLIC_KEY; import static com.hedera.node.app.service.entityid.impl.schemas.V0590EntityIdSchema.ENTITY_COUNTS_STATE_ID; +import static com.hedera.node.app.service.roster.impl.RosterTransitionWeights.atLeastOneThirdOfTotal; import static com.hedera.services.bdd.junit.hedera.ExternalPath.APPLICATION_PROPERTIES; import static com.hedera.services.bdd.junit.hedera.ExternalPath.DATA_CONFIG_DIR; import static com.hedera.services.bdd.junit.hedera.ExternalPath.SAVED_STATES_DIR; @@ -32,6 +33,7 @@ import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toMap; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import com.hedera.hapi.block.stream.Block; @@ -166,6 +168,7 @@ public class StateChangesValidator implements BlockStreamValidator { /** * The relevant context from a history proof construction. + * * @param proverWeights the weights of the nodes in the prover history roster * @param targetWeights the weights of the nodes in the target history roster * @param proofKeys the proof keys of the nodes in the target history roster @@ -659,74 +662,65 @@ private void validateBlockProof( provenHash = combine(provenHash, siblingHash.siblingHash()); } } - - // TODO: verify hints proof - // if (hintsLibrary != null) { - // final var signature = proof.blockSignature(); - // final var vk = proof.verificationKey(); - // final boolean valid = hintsLibrary.verifyAggregate(signature, provenHash, vk, 1, - // hintsThresholdDenominator); - // if (!valid) { - // Assertions.fail(() -> "Invalid signature in proof (start round #" + firstRound + ") - " + - // proof); - // } else { - // logger.info("Verified signature on #{}", proof.block()); - // } - // if (historyLibrary != null) { - // assertTrue( - // proof.hasVerificationKeyProof(), - // "No chain-of-trust for hinTS key in proof (start round #" + firstRound + ") - " + - // proof); - // final var chainOfTrustProof = proof.verificationKeyProofOrThrow(); - // switch (chainOfTrustProof.proof().kind()) { - // case UNSET -> - // Assertions.fail("Empty chain-of-trust for hinTS key in proof (start round #" + - // firstRound - // + ") - " + proof); - // case NODE_SIGNATURES -> { - // requireNonNull(activeWeights); - // final var context = vkContexts.get(vk); - // assertNotNull( - // context, "No context for verification key in proof (start round #" + - // firstRound + ")"); - // // Signatures are over (targetBookHash || hash(verificationKey)) - // final var targetBookHash = context.targetBookHash(historyLibrary); - // final var message = - // targetBookHash.append(historyLibrary.hashHintsVerificationKey(vk)); - // long signingWeight = 0; - // final var signatures = - // chainOfTrustProof.nodeSignaturesOrThrow().nodeSignatures(); - // final var weights = context.proverWeights(); - // for (final var s : signatures) { - // final long nodeId = s.nodeId(); - // final var proofKey = context.proofKeys().get(nodeId); - // assertTrue( - // historyLibrary.verifySchnorr(s.signature(), message, proofKey), - // "Invalid signature for node" + nodeId - // + " in chain-of-trust for hinTS key in proof (start round #" + - // firstRound - // + ") - " + proof); - // signingWeight += weights.getOrDefault(s.nodeId(), 0L); - // } - // final long threshold = atLeastOneThirdOfTotal(weights); - // assertTrue( - // signingWeight >= threshold, - // "Insufficient weight in chain-of-trust for hinTS key in proof (start round #" - // + firstRound + ") - " + proof - // + " (expected >= " + threshold + ", got " + signingWeight - // + ")"); - // } - // case WRAPS_PROOF -> - // assertTrue( - // historyLibrary.verifyChainOfTrust(chainOfTrustProof.wrapsProofOrThrow()), - // "Insufficient weight in chain-of-trust for hinTS key in proof (start round #" - // + firstRound + ") - " + proof); - // } - // } - // } else { - // final var expectedSignature = Bytes.wrap(noThrowSha384HashOf(provenHash.toByteArray())); - // assertEquals(expectedSignature, proof.blockSignature(), "Signature mismatch for " + proof); - // } + if (hintsLibrary != null) { + final var signature = proof.signedBlockProof().blockSignature(); + final var vk = proof.verificationKey(); + final boolean valid = hintsLibrary.verifyAggregate(signature, provenHash, vk, 1, hintsThresholdDenominator); + if (!valid) { + Assertions.fail(() -> "Invalid signature in proof (start round #" + firstRound + ") - " + proof); + } else { + logger.info("Verified signature on #{}", proof.block()); + } + if (historyLibrary != null) { + assertTrue( + proof.hasVerificationKeyProof(), + "No chain-of-trust for hinTS key in proof (start round #" + firstRound + ") - " + proof); + final var chainOfTrustProof = proof.verificationKeyProofOrThrow(); + switch (chainOfTrustProof.proof().kind()) { + case UNSET -> + Assertions.fail("Empty chain-of-trust for hinTS key in proof (start round #" + firstRound + + ") - " + proof); + case NODE_SIGNATURES -> { + requireNonNull(activeWeights); + final var context = vkContexts.get(vk); + assertNotNull( + context, "No context for verification key in proof (start round #" + firstRound + ")"); + // Signatures are over (targetBookHash || hash(verificationKey)) + final var targetBookHash = context.targetBookHash(historyLibrary); + final var message = targetBookHash.append(historyLibrary.hashHintsVerificationKey(vk)); + long signingWeight = 0; + final var signatures = + chainOfTrustProof.nodeSignaturesOrThrow().nodeSignatures(); + final var weights = context.proverWeights(); + for (final var s : signatures) { + final long nodeId = s.nodeId(); + final var proofKey = context.proofKeys().get(nodeId); + assertTrue( + historyLibrary.verifySchnorr(s.signature(), message, proofKey), + "Invalid signature for node" + nodeId + + " in chain-of-trust for hinTS key in proof (start round #" + firstRound + + ") - " + proof); + signingWeight += weights.getOrDefault(s.nodeId(), 0L); + } + final long threshold = atLeastOneThirdOfTotal(weights); + assertTrue( + signingWeight >= threshold, + "Insufficient weight in chain-of-trust for hinTS key in proof (start round #" + + firstRound + ") - " + proof + + " (expected >= " + threshold + ", got " + signingWeight + + ")"); + } + case WRAPS_PROOF -> + assertTrue( + historyLibrary.verifyChainOfTrust(chainOfTrustProof.wrapsProofOrThrow()), + "Insufficient weight in chain-of-trust for hinTS key in proof (start round #" + + firstRound + ") - " + proof); + } + } + } else { + final var expectedSignature = Bytes.wrap(noThrowSha384HashOf(provenHash.toByteArray())); + assertEquals(expectedSignature, proof.blockSignature(), "Signature mismatch for " + proof); + } } private String rootMnemonicFor(@NonNull final MerkleNode state) { From c204d91c75b088f2f96714e2b67baff701525c7e Mon Sep 17 00:00:00 2001 From: Neeharika-Sompalli Date: Sun, 2 Nov 2025 16:56:26 -0600 Subject: [PATCH 46/63] validation passes Signed-off-by: Neeharika-Sompalli --- .../validators/block/StateChangesValidator.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java index aaed04019b18..2bfcd85331fc 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java @@ -211,12 +211,12 @@ public static void main(String[] args) { final long realm = 12; final var validator = new StateChangesValidator( Bytes.fromHex( - "bd2ee10d715acd195587977ee3f259e68cc84a55cc593a9789b36ef91aee3548434dec3149e4bf5ddecc020e0fbf1193"), + "50ea5c2588457b952dba215bcefc5f54a1b87c298e5c0f2a534a8eb7177354126c55ee5c23319187e964443e4c17c007"), node0Dir.resolve("output/swirlds.log"), node0Dir.resolve("data/config/application.properties"), node0Dir.resolve("data/config"), 16, - HintsEnabled.NO, + HintsEnabled.YES, HistoryEnabled.NO, hintsThresholdDenominator, shard, @@ -661,9 +661,14 @@ private void validateBlockProof( // Our indirect proofs always provide right sibling hashes provenHash = combine(provenHash, siblingHash.siblingHash()); } + // FUTURE: When Merkle Paths are populated, stop returning and verify indirect proofs + return; } if (hintsLibrary != null) { - final var signature = proof.signedBlockProof().blockSignature(); + if (!proof.hasSignedBlockProof()) { + return; + } + final var signature = proof.signedBlockProofOrThrow().blockSignature(); final var vk = proof.verificationKey(); final boolean valid = hintsLibrary.verifyAggregate(signature, provenHash, vk, 1, hintsThresholdDenominator); if (!valid) { From e5bded6fea216f694704a8e90066e60149c25aca Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sun, 2 Nov 2025 16:16:59 -0700 Subject: [PATCH 47/63] Proto updates Signed-off-by: Matt Hess --- .../main/proto/network/pending_proof.proto | 52 ++++++++-------- .../main/proto/block/stream/state_proof.proto | 1 - .../state/blockstream/merkle_leaf.proto | 35 ++++++----- .../blockstream/streaming_tree_snapshot.proto | 61 ++++++++++--------- 4 files changed, 74 insertions(+), 75 deletions(-) diff --git a/hapi/hapi/src/main/proto/network/pending_proof.proto b/hapi/hapi/src/main/proto/network/pending_proof.proto index 442f996ebf38..65f8fdf1013a 100644 --- a/hapi/hapi/src/main/proto/network/pending_proof.proto +++ b/hapi/hapi/src/main/proto/network/pending_proof.proto @@ -25,30 +25,30 @@ option java_multiple_files = true; * Provides context for a block proof pending a TSS signature. */ message PendingProof { - /** - * The block whose proof is pending. - */ - uint64 block = 1; - /** - * The hash requiring a proof. - */ - bytes block_hash = 2; - /** - * The state hash of the start of the pending block. - */ - bytes start_of_block_state_root_hash = 3; - /** - * The hash of the previous block. - */ - bytes previous_block_hash = 4; - /** - * If set, the sibling hashes that could be used to prove the - * previous block hash (in case it was also pending, and we - * obtain a signature for this block first). - *

- * If not set, the previous block hash can be assumed to have - * already been signed and this block proof will not be used - * for any indirect proofs. - */ - repeated com.hedera.hapi.block.stream.MerkleSiblingHash sibling_hashes_from_prev_block_root = 5; + /** + * The block whose proof is pending. + */ + uint64 block = 1; + /** + * The hash requiring a signature. + */ + bytes block_hash = 2; + /** + * The state hash of the start of the pending block. + */ + bytes start_of_block_state_root_hash = 3; + /** + * The hash of the previous block. + */ + bytes previous_block_hash = 4; + /** + * If set, the sibling hashes that could be used to prove the + * previous block hash (in case it was also pending, and we + * obtain a signature for this block first). + *

+ * If not set, the previous block hash can be assumed to have + * already been signed and this block proof will not be used + * for any indirect proofs. + */ + repeated com.hedera.hapi.block.stream.MerkleSiblingHash sibling_hashes_from_prev_block_root = 5; } diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto index 163dd5f762e0..92e7edf04e83 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/state_proof.proto @@ -155,4 +155,3 @@ message SiblingNode { */ bytes hash = 2; } - diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto index 8b9b690c3d28..2f7f2e8bcca6 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto @@ -22,27 +22,26 @@ option java_package = "com.hedera.hapi.block.stream.protoc"; // <<>> This comment is special code for setting PBJ Compiler java package option java_multiple_files = true; -/** - * TODO - */ message MerkleLeaf { - oneof content { /** - * TODO + * The content of this merkle leaf node. */ - proto.Timestamp block_consensus_timestamp = 1; + oneof content { + /** + * The consensus timestamp of the first round in the current + * block, which is equivalent to the consensus timestamp of + * the first transaction in that round. + */ + bytes block_consensus_timestamp = 1; - /** - * TODO – SHOULD BE BlockItem, not bytes - */ - // com.hedera.hapi.block.stream.BlockItem block_item = 2; - bytes block_item = 2; + /** + * The serialized bytes of a single BlockItem message. + */ + bytes block_item = 2; - /** - * TODO – SHOULD BE StateItem, not bytes - */ - // com.hedera.hapi.platform.state.StateItem state_item = 3; - bytes state_item = 3; - } + /** + * The serialized bytes of a single StateItem message. + */ + bytes state_item = 3; + } } - diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/streaming_tree_snapshot.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/streaming_tree_snapshot.proto index aa6308f6db0e..fce38c8b2d28 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/streaming_tree_snapshot.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/streaming_tree_snapshot.proto @@ -1,5 +1,8 @@ /** - * TODO + * # Streaming Tree Snapshot + * A complete snapshot of an incremental streaming merkle subtree. + * This message contains the hashes necessary to reconstruct all + * of the subtree's 'uncollapsed' nodes. * * ### Keywords * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", @@ -17,39 +20,37 @@ option java_package = "com.hedera.hapi.block.stream.protoc"; // <<>> This comment is special code for setting PBJ Compiler java package option java_multiple_files = true; -/** - * TODO - */ -message StreamingTreeSnapshot { +message IncrementalStreamingSnapshot { - /** - * Which of the block merkle sub trees this snapshot represents - */ - SubMerkleTree type = 1; + /** + * Which of the block merkle sub trees this snapshot represents + */ + SubMerkleTree type = 1; - /** - * All the uncollapsed nodes of the sub tree - */ - repeated bytes nodes = 2; + /** + * All the uncollapsed nodes of the sub tree. These nodes SHALL + * be ordered from left to right and bottom to top. + */ + repeated bytes nodes = 2; } /** Identifer for each sub-tree of the block root fixed size tree */ enum SubMerkleTree { - ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice - PREVIOUS_BLOCK_ROOT = 1; - PREVIOUS_ROOTS_TREE = 2; - PREVIOUS_BLOCK_START_STATE = 3; - CONSENSUS_HEADER_ITEMS = 4; - INPUT_ITEMS_TREE = 5; - OUTPUT_ITEMS_TREE = 6; - STATE_CHANGE_ITEMS_TREE = 7; - TRACE_DATA_ITEMS_TREE = 8; - FUTURE_1 = 9; // these place holders for future use sub trees, will be renamed if they are used later - FUTURE_2 = 10; - FUTURE_3 = 11; - FUTURE_4 = 12; - FUTURE_5 = 13; - FUTURE_6 = 14; - FUTURE_7 = 15; - FUTURE_8 = 16; + ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice + PREVIOUS_BLOCK_ROOT = 1; + PREVIOUS_ROOTS_TREE = 2; + PREVIOUS_BLOCK_START_STATE = 3; + CONSENSUS_HEADER_ITEMS = 4; + INPUT_ITEMS_TREE = 5; + OUTPUT_ITEMS_TREE = 6; + STATE_CHANGE_ITEMS_TREE = 7; + TRACE_DATA_ITEMS_TREE = 8; + FUTURE_1 = 9; // these place holders for future use sub trees, will be renamed if they are used later + FUTURE_2 = 10; + FUTURE_3 = 11; + FUTURE_4 = 12; + FUTURE_5 = 13; + FUTURE_6 = 14; + FUTURE_7 = 15; + FUTURE_8 = 16; } From 78989509963ff9510d514fcd47b902d03d161802 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sun, 2 Nov 2025 16:57:53 -0700 Subject: [PATCH 48/63] Revert merkle leaf to proto.Timestamp Signed-off-by: Matt Hess --- .../src/main/proto/services/state/blockstream/merkle_leaf.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto index 2f7f2e8bcca6..72f235095d81 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto @@ -32,7 +32,7 @@ message MerkleLeaf { * block, which is equivalent to the consensus timestamp of * the first transaction in that round. */ - bytes block_consensus_timestamp = 1; + proto.Timestamp block_consensus_timestamp = 1; /** * The serialized bytes of a single BlockItem message. From e6797ff2098cda4fc94739b0dc8e5119fad5b3a1 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sun, 2 Nov 2025 17:31:39 -0700 Subject: [PATCH 49/63] More proto doc updates, renumbering Signed-off-by: Matt Hess --- .../state/blockstream/block_stream_info.proto | 256 +++++++++--------- 1 file changed, 128 insertions(+), 128 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto index 8f8b6eb6c7de..3dc23b617ffb 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto @@ -39,11 +39,11 @@ option java_multiple_files = true; * block item in this block. */ message BlockStreamInfo { - /** - * A block number.
- * This is the current block number. - */ - uint64 block_number = 1; + /** + * A block number.
+ * This is the current block number. + */ + uint64 block_number = 1; /** * A consensus time for the current block.
@@ -56,31 +56,31 @@ message BlockStreamInfo { */ proto.Timestamp block_time = 2; - /** - * A concatenation of hash values.
- * This combines several trailing output block item hashes and - * is used as a seed value for a pseudo-random number generator.
- * This is also required to implement the EVM `PREVRANDAO` opcode.
- * This MUST contain at least 256 bits of entropy. - */ - bytes trailing_output_hashes = 3; - - /** - * A concatenation of hash values.
- * This field combines up to 256 trailing block hashes. - *

- * If this message is for block number N, then the earliest available - * hash SHALL be for block number N-256.
- * The latest available hash SHALL be for block N-1.
- * This is REQUIRED to implement the EVM `BLOCKHASH` opcode. - *

- * ### Field Length - * Each hash value SHALL be the trailing 265 bits of a SHA2-384 hash.
- * The length of this field SHALL be an integer multiple of 32 bytes.
- * This field SHALL be at least 32 bytes.
- * The maximum length of this field SHALL be 8192 bytes. - */ - bytes trailing_block_hashes = 4; + /** + * A concatenation of hash values.
+ * This combines several trailing output block item hashes and + * is used as a seed value for a pseudo-random number generator.
+ * This is also required to implement the EVM `PREVRANDAO` opcode.
+ * This MUST contain at least 256 bits of entropy. + */ + bytes trailing_output_hashes = 3; + + /** + * A concatenation of hash values.
+ * This field combines up to 256 trailing block hashes. + *

+ * If this message is for block number N, then the earliest available + * hash SHALL be for block number N-256.
+ * The latest available hash SHALL be for block N-1.
+ * This is REQUIRED to implement the EVM `BLOCKHASH` opcode. + *

+ * ### Field Length + * Each hash value SHALL be the trailing 265 bits of a SHA2-384 hash.
+ * The length of this field SHALL be an integer multiple of 32 bytes.
+ * This field SHALL be at least 32 bytes.
+ * The maximum length of this field SHALL be 8192 bytes. + */ + bytes trailing_block_hashes = 4; /** * A SHA2-384 hash value.
@@ -88,102 +88,102 @@ message BlockStreamInfo { */ bytes input_tree_root_hash = 5; - /** - * A SHA2-384 hash value.
- * This is the hash of consensus state at the _start_ of this block. - */ - bytes start_of_block_state_hash = 6; - - /** - * A count of "output" block items in this block. - *

- * This SHALL count the number of output block items that _precede_ - * the state change that updates this singleton. - */ - uint32 num_preceding_state_changes_items = 7; - - /** - * A concatenation of SHA2-384 hash values.
- * This is the "rightmost" values of the "output" subtree. - *

- * The subtree containing these hashes SHALL be constructed from all "output" - * `BlockItem`s in this block that _precede_ the update to this singleton. - */ - repeated bytes rightmost_preceding_state_changes_tree_hashes = 8; - - /** - * A block-end consensus time stamp. - *

- * This field SHALL hold the last-used consensus time for - * the current block. - */ - proto.Timestamp block_end_time = 9; - - /** - * Whether the post-upgrade work has been done. - *

- * This MUST be false if and only if the network just restarted - * after an upgrade and has not yet done the post-upgrade work. - */ - bool post_upgrade_work_done = 10; - - /** - * A version describing the version of application software. - *

- * This SHALL be the software version that created this block. - */ - proto.SemanticVersion creation_software_version = 11; - - /** - * The time stamp at which the last interval process was done. - *

- * This field SHALL hold the consensus time for the last time - * at which an interval of time-dependent events were processed. - */ - proto.Timestamp last_interval_process_time = 12; - - /** - * The time stamp at which the last user transaction was handled. - *

- * This field SHALL hold the consensus time for the last time - * at which a user transaction was handled. - */ - proto.Timestamp last_handle_time = 13; - - /** - * A SHA2-384 hash value.
- * This is the final hash of the "input" subtree for this block. - */ - bytes consensus_header_root_hash = 14; - - /** - * A SHA2-384 hash value.
- * This is the final hash of the "input" subtree for this block. - */ - bytes output_item_root_hash = 15; - - /** - * A SHA2-384 hash value.
- * This is the final hash of the "trace data" subtree for this block. - */ - bytes trace_data_root_hash = 17; - - /** - * The time stamp at which the first transaction was handled in - * this block. - */ - proto.Timestamp block_start_consensus_timestamp = 18; - - /** - * The intermediate hashes needed for branch 2 in the block merkle - * tree structure. These hashes SHALL include all block root hashes - * needed to construct branch 2's final state at the end of the - * previous block - */ - repeated bytes intermediate_previous_block_root_hashes = 19; - - /** - * The number of leaves in the intermediate block roots subtree. - */ - uint64 intermediate_block_roots_leaf_count = 20; + /** + * A SHA2-384 hash value.
+ * This is the hash of consensus state at the _start_ of this block. + */ + bytes start_of_block_state_hash = 6; + + /** + * A count of "output" block items in this block. + *

+ * This SHALL count the number of output block items that _precede_ + * the state change that updates this singleton. + */ + uint32 num_preceding_state_changes_items = 7; + + /** + * A concatenation of SHA2-384 hash values.
+ * This is the "rightmost" values of the "output" subtree. + *

+ * The subtree containing these hashes SHALL be constructed from all "output" + * `BlockItem`s in this block that _precede_ the update to this singleton. + */ + repeated bytes rightmost_preceding_state_changes_tree_hashes = 8; + + /** + * A block-end consensus time stamp. + *

+ * This field SHALL hold the last-used consensus time for + * the current block. + */ + proto.Timestamp block_end_time = 9; + + /** + * Whether the post-upgrade work has been done. + *

+ * This MUST be false if and only if the network just restarted + * after an upgrade and has not yet done the post-upgrade work. + */ + bool post_upgrade_work_done = 10; + + /** + * A version describing the version of application software. + *

+ * This SHALL be the software version that created this block. + */ + proto.SemanticVersion creation_software_version = 11; + + /** + * The time stamp at which the last interval process was done. + *

+ * This field SHALL hold the consensus time for the last time + * at which an interval of time-dependent events were processed. + */ + proto.Timestamp last_interval_process_time = 12; + + /** + * The time stamp at which the last user transaction was handled. + *

+ * This field SHALL hold the consensus time for the last time + * at which a user transaction was handled. + */ + proto.Timestamp last_handle_time = 13; + + /** + * A SHA2-384 hash value.
+ * This is the final hash of the "consensus headers" subtree for this block. + */ + bytes consensus_header_root_hash = 14; + + /** + * A SHA2-384 hash value.
+ * This is the final hash of the "output" subtree for this block. + */ + bytes output_item_root_hash = 15; + + /** + * A SHA2-384 hash value.
+ * This is the final hash of the "trace data" subtree for this block. + */ + bytes trace_data_root_hash = 16; + + /** + * The time stamp at which the first transaction was handled in + * this block. + */ + proto.Timestamp block_start_consensus_timestamp = 17; + + /** + * The intermediate hashes needed for subroot 2 in the block merkle + * tree structure. These hashes SHALL include the minimum required + * block root hashes needed to construct subroot 2's final state at + * the end of the previous block. + */ + repeated bytes intermediate_previous_block_root_hashes = 19; + + /** + * The number of leaves in the intermediate block roots subtree. + */ + uint64 intermediate_block_roots_leaf_count = 20; } From 4136cb6e2a6aad2b9e5a1d87ab5197f7c40ea7c5 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sun, 2 Nov 2025 17:34:38 -0700 Subject: [PATCH 50/63] Try removing duplicate timestamp Signed-off-by: Matt Hess --- .../hedera/node/app/blocks/impl/BlockStreamManagerImpl.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index d526339026a2..a090ebf65209 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -148,7 +148,6 @@ public class BlockStreamManagerImpl implements BlockStreamManager { private Instant consensusTimeLastRound; private Timestamp lastUsedTime; private BlockItemWriter writer; - private Instant firstConsensusTimeOfCurrentBlock; // Block merkle subtrees and leaves private IncrementalStreamingHasher previousBlockHashes; @@ -393,7 +392,6 @@ public void startRound(@NonNull final Round round, @NonNull final State state) { .blockTimestamp(asTimestamp(blockTimestamp)) .hapiProtoVersion(hapiVersion); worker.addItem(BlockItem.newBuilder().blockHeader(header).build()); - firstConsensusTimeOfCurrentBlock = round.getConsensusTimestamp(); } consensusTimeLastRound = round.getConsensusTimestamp(); } @@ -581,7 +579,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { consensusHeaderHash, outputsHash, traceDataHash, - asTimestamp(firstConsensusTimeOfCurrentBlock), + asTimestamp(blockTimestamp), previousBlockHashes.intermediateHashingState(), previousBlockHashes.leafCount()); blockStreamInfoState.put(newBlockStreamInfo); From 15259880e0e03ef6fc0adc5f3c0c1bb06ee2dfb2 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sun, 2 Nov 2025 18:07:15 -0700 Subject: [PATCH 51/63] Spotless Signed-off-by: Matt Hess --- .../com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index a090ebf65209..5930794e47f2 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -579,7 +579,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { consensusHeaderHash, outputsHash, traceDataHash, - asTimestamp(blockTimestamp), + asTimestamp(blockTimestamp), previousBlockHashes.intermediateHashingState(), previousBlockHashes.leafCount()); blockStreamInfoState.put(newBlockStreamInfo); From 42611d4900d74ec28aaa7b72a60f60f0e9bf2e59 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sun, 2 Nov 2025 18:38:41 -0700 Subject: [PATCH 52/63] Use block proof's TSS signature Signed-off-by: Matt Hess --- .../support/validators/block/StateChangesValidator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java index 2bfcd85331fc..55389db8d063 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java @@ -354,9 +354,8 @@ public void validateBlocks(@NonNull final List blocks) { .filter(HintsConstruction::hasHintsScheme) .forEach(c -> preprocessedKeys.put( c.constructionId(), c.hintsSchemeOrThrow().preprocessedKeysOrThrow())); - final IncrementalStreamingHasher incrementalBlockHashes = - new IncrementalStreamingHasher(CommonUtils.sha384DigestOrThrow(), List.of(), 0); - incrementalBlockHashes.addLeaf(BlockStreamManager.ZERO_BLOCK_HASH.toByteArray()); + final IncrementalStreamingHasher incrementalBlockHashes = new IncrementalStreamingHasher( + CommonUtils.sha384DigestOrThrow(), List.of(BlockStreamManager.ZERO_BLOCK_HASH.toByteArray()), 1); for (int i = 0; i < n; i++) { final var block = blocks.get(i); final var shouldVerifyProof = @@ -724,7 +723,8 @@ private void validateBlockProof( } } else { final var expectedSignature = Bytes.wrap(noThrowSha384HashOf(provenHash.toByteArray())); - assertEquals(expectedSignature, proof.blockSignature(), "Signature mismatch for " + proof); + assertEquals( + expectedSignature, proof.signedBlockProof().blockSignature(), "Signature mismatch for " + proof); } } From 73416cea1bfeea15a6dfdecfc53052722d1892ed Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sun, 2 Nov 2025 19:13:20 -0700 Subject: [PATCH 53/63] Restore inc hashes init in validator Signed-off-by: Matt Hess --- .../support/validators/block/StateChangesValidator.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java index 55389db8d063..61d2d06e8d80 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/StateChangesValidator.java @@ -354,8 +354,9 @@ public void validateBlocks(@NonNull final List blocks) { .filter(HintsConstruction::hasHintsScheme) .forEach(c -> preprocessedKeys.put( c.constructionId(), c.hintsSchemeOrThrow().preprocessedKeysOrThrow())); - final IncrementalStreamingHasher incrementalBlockHashes = new IncrementalStreamingHasher( - CommonUtils.sha384DigestOrThrow(), List.of(BlockStreamManager.ZERO_BLOCK_HASH.toByteArray()), 1); + final IncrementalStreamingHasher incrementalBlockHashes = + new IncrementalStreamingHasher(CommonUtils.sha384DigestOrThrow(), List.of(), 0); + incrementalBlockHashes.addLeaf(BlockStreamManager.ZERO_BLOCK_HASH.toByteArray()); for (int i = 0; i < n; i++) { final var block = blocks.get(i); final var shouldVerifyProof = From 1d3f6b6f49b14b3783784210dcf0bee499719c06 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sun, 2 Nov 2025 19:44:17 -0700 Subject: [PATCH 54/63] Copy array refs in inc hasher Signed-off-by: Matt Hess --- .../node/app/blocks/impl/IncrementalStreamingHasher.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java index bd9715793957..f30b65fdc391 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java @@ -3,6 +3,7 @@ import com.hedera.pbj.runtime.io.buffer.Bytes; import java.security.MessageDigest; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -81,7 +82,9 @@ public byte[] computeRootHash() { */ public List intermediateHashingState() { // do we need to copy the arrays here so they don't change? - return hashList.stream().map(Bytes::wrap).toList(); + return hashList.stream() + .map(b -> Bytes.wrap(Arrays.copyOf(b, b.length))) + .toList(); } /** From 5b6fc983137a09b467ea1a4726277e457dad1c26 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sun, 2 Nov 2025 21:49:44 -0700 Subject: [PATCH 55/63] Remove duplicate timestamp from block stream info Signed-off-by: Matt Hess --- .../services/state/blockstream/block_stream_info.proto | 10 ++-------- .../node/app/blocks/impl/BlockStreamManagerImpl.java | 5 ++--- .../app/blocks/impl/IncrementalStreamingHasher.java | 1 - 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto index 3dc23b617ffb..8210c641e6d4 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/block_stream_info.proto @@ -168,22 +168,16 @@ message BlockStreamInfo { */ bytes trace_data_root_hash = 16; - /** - * The time stamp at which the first transaction was handled in - * this block. - */ - proto.Timestamp block_start_consensus_timestamp = 17; - /** * The intermediate hashes needed for subroot 2 in the block merkle * tree structure. These hashes SHALL include the minimum required * block root hashes needed to construct subroot 2's final state at * the end of the previous block. */ - repeated bytes intermediate_previous_block_root_hashes = 19; + repeated bytes intermediate_previous_block_root_hashes = 17; /** * The number of leaves in the intermediate block roots subtree. */ - uint64 intermediate_block_roots_leaf_count = 20; + uint64 intermediate_block_roots_leaf_count = 18; } diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 5930794e47f2..5ea96c9d9a28 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -335,7 +335,7 @@ public void init(@NonNull final State state, @Nullable final Bytes lastBlockHash blockStreamInfo.outputItemRootHash(), lastBlockFinalStateChangesHash, blockStreamInfo.traceDataRootHash(), - blockStreamInfo.blockStartConsensusTimestamp()) + blockStreamInfo.blockTime()) .blockRootHash()); requireNonNull(calculatedLastBlockHash); this.lastBlockHash = calculatedLastBlockHash; @@ -579,7 +579,6 @@ public boolean endRound(@NonNull final State state, final long roundNum) { consensusHeaderHash, outputsHash, traceDataHash, - asTimestamp(blockTimestamp), previousBlockHashes.intermediateHashingState(), previousBlockHashes.leafCount()); blockStreamInfoState.put(newBlockStreamInfo); @@ -601,7 +600,7 @@ public boolean endRound(@NonNull final State state, final long roundNum) { outputsHash, stateChangesHash, traceDataHash, - newBlockStreamInfo.blockStartConsensusTimestamp()); + newBlockStreamInfo.blockTime()); final var finalBlockRootHash = rootAndSiblingHashes.blockRootHash(); // Create BlockFooter with the three essential hashes: diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java index f30b65fdc391..59a267031cb8 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/IncrementalStreamingHasher.java @@ -81,7 +81,6 @@ public byte[] computeRootHash() { * @return the intermediate hashing state */ public List intermediateHashingState() { - // do we need to copy the arrays here so they don't change? return hashList.stream() .map(b -> Bytes.wrap(Arrays.copyOf(b, b.length))) .toList(); From 72ee0d661af42c703718e95c4127a15726ece76a Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sun, 2 Nov 2025 22:20:59 -0700 Subject: [PATCH 56/63] Fix unit tests, remove unused proto Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 24 +- .../state/blockstream/merkle_leaf.proto | 2 +- .../blockstream/streaming_tree_snapshot.proto | 56 -- .../blocks/impl/BlockStreamManagerImpl.java | 2 +- .../impl/BlockStreamManagerImplTest.java | 543 +++++++++--------- 5 files changed, 311 insertions(+), 316 deletions(-) delete mode 100644 hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/streaming_tree_snapshot.proto diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index 09a709be5e29..b41749642be1 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -38,7 +38,6 @@ import "block/stream/output/transaction_output.proto"; import "block/stream/output/transaction_result.proto"; import "block/stream/trace/trace_data.proto"; import "block/stream/output/block_footer.proto"; -import "services/state/blockstream/streaming_tree_snapshot.proto"; /** * A single item within a block stream. @@ -390,3 +389,26 @@ message RedactedItem { */ SubMerkleTree tree = 3; } + +/** + * Identifier for each sub-tree of the block root fixed size tree + */ +enum SubMerkleTree { + ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice + PREVIOUS_BLOCK_ROOT = 1; + PREVIOUS_ROOTS_TREE = 2; + PREVIOUS_BLOCK_START_STATE = 3; + CONSENSUS_HEADER_ITEMS = 4; + INPUT_ITEMS_TREE = 5; + OUTPUT_ITEMS_TREE = 6; + STATE_CHANGE_ITEMS_TREE = 7; + TRACE_DATA_ITEMS_TREE = 8; + FUTURE_1 = 9; // these place holders for future use sub trees, will be renamed if they are used later + FUTURE_2 = 10; + FUTURE_3 = 11; + FUTURE_4 = 12; + FUTURE_5 = 13; + FUTURE_6 = 14; + FUTURE_7 = 15; + FUTURE_8 = 16; +} diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto index 72f235095d81..2f7f2e8bcca6 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/merkle_leaf.proto @@ -32,7 +32,7 @@ message MerkleLeaf { * block, which is equivalent to the consensus timestamp of * the first transaction in that round. */ - proto.Timestamp block_consensus_timestamp = 1; + bytes block_consensus_timestamp = 1; /** * The serialized bytes of a single BlockItem message. diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/streaming_tree_snapshot.proto b/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/streaming_tree_snapshot.proto deleted file mode 100644 index fce38c8b2d28..000000000000 --- a/hapi/hedera-protobuf-java-api/src/main/proto/services/state/blockstream/streaming_tree_snapshot.proto +++ /dev/null @@ -1,56 +0,0 @@ -/** - * # Streaming Tree Snapshot - * A complete snapshot of an incremental streaming merkle subtree. - * This message contains the hashes necessary to reconstruct all - * of the subtree's 'uncollapsed' nodes. - * - * ### Keywords - * The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", - * "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this - * document are to be interpreted as described in - * [RFC2119](https://www.ietf.org/rfc/rfc2119) and clarified in - * [RFC8174](https://www.ietf.org/rfc/rfc8174). - */ -syntax = "proto3"; - -package com.hedera.hapi.block.stream; - -// SPDX-License-Identifier: Apache-2.0 -option java_package = "com.hedera.hapi.block.stream.protoc"; -// <<>> This comment is special code for setting PBJ Compiler java package -option java_multiple_files = true; - -message IncrementalStreamingSnapshot { - - /** - * Which of the block merkle sub trees this snapshot represents - */ - SubMerkleTree type = 1; - - /** - * All the uncollapsed nodes of the sub tree. These nodes SHALL - * be ordered from left to right and bottom to top. - */ - repeated bytes nodes = 2; -} - -/** Identifer for each sub-tree of the block root fixed size tree */ -enum SubMerkleTree { - ITEM_TYPE_UNSPECIFIED = 0; // Default value, required best practice - PREVIOUS_BLOCK_ROOT = 1; - PREVIOUS_ROOTS_TREE = 2; - PREVIOUS_BLOCK_START_STATE = 3; - CONSENSUS_HEADER_ITEMS = 4; - INPUT_ITEMS_TREE = 5; - OUTPUT_ITEMS_TREE = 6; - STATE_CHANGE_ITEMS_TREE = 7; - TRACE_DATA_ITEMS_TREE = 8; - FUTURE_1 = 9; // these place holders for future use sub trees, will be renamed if they are used later - FUTURE_2 = 10; - FUTURE_3 = 11; - FUTURE_4 = 12; - FUTURE_5 = 13; - FUTURE_6 = 14; - FUTURE_7 = 15; - FUTURE_8 = 16; -} diff --git a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java index 5ea96c9d9a28..e28dabd44839 100644 --- a/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java +++ b/hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImpl.java @@ -29,6 +29,7 @@ import com.hedera.hapi.block.stream.MerklePath; import com.hedera.hapi.block.stream.MerkleSiblingHash; import com.hedera.hapi.block.stream.StateProof; +import com.hedera.hapi.block.stream.SubMerkleTree; import com.hedera.hapi.block.stream.TssSignedBlockProof; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.block.stream.output.SingletonUpdateChange; @@ -37,7 +38,6 @@ import com.hedera.hapi.node.base.SemanticVersion; import com.hedera.hapi.node.base.Timestamp; import com.hedera.hapi.node.state.blockstream.BlockStreamInfo; -import com.hedera.hapi.node.state.blockstream.SubMerkleTree; import com.hedera.hapi.platform.state.PlatformState; import com.hedera.node.app.blocks.BlockHashSigner; import com.hedera.node.app.blocks.BlockItemWriter; diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index b756d063df4d..8099051cd02c 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -7,12 +7,15 @@ import static com.hedera.node.app.blocks.BlockStreamManager.ZERO_BLOCK_HASH; import static com.hedera.node.app.blocks.BlockStreamService.FAKE_RESTART_BLOCK_HASH; import static com.hedera.node.app.blocks.impl.BlockImplUtils.appendHash; +import static com.hedera.node.app.blocks.impl.BlockImplUtils.combine; import static com.hedera.node.app.blocks.schemas.V0560BlockStreamSchema.BLOCK_STREAM_INFO_STATE_ID; import static com.hedera.node.app.blocks.schemas.V0560BlockStreamSchema.BLOCK_STREAM_INFO_STATE_LABEL; import static com.hedera.node.app.fixtures.AppTestBase.DEFAULT_CONFIG; import static com.hedera.node.app.hapi.utils.CommonUtils.noThrowSha384HashOf; import static com.swirlds.platform.state.service.schemas.V0540PlatformStateSchema.PLATFORM_STATE_STATE_ID; import static com.swirlds.platform.state.service.schemas.V0540PlatformStateSchema.PLATFORM_STATE_STATE_LABEL; +import static com.swirlds.platform.test.fixtures.state.TestPlatformStateFacade.TEST_PLATFORM_STATE_FACADE; +import static java.time.Instant.EPOCH; import static java.util.concurrent.CompletableFuture.completedFuture; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -33,6 +36,8 @@ import static org.mockito.Mockito.withSettings; import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.stream.ChainOfTrustProof; +import com.hedera.hapi.block.stream.RecordFileItem; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.block.stream.output.StateChanges; import com.hedera.hapi.block.stream.output.TransactionResult; @@ -70,18 +75,21 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import org.bouncycastle.util.Arrays; import org.hiero.base.crypto.Hash; import org.hiero.base.crypto.test.fixtures.CryptoRandomUtils; import org.hiero.consensus.model.event.ConsensusEvent; import org.hiero.consensus.model.hashgraph.Round; +import org.hiero.consensus.model.transaction.ConsensusTransaction; +import org.hiero.consensus.model.transaction.TransactionWrapper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -112,8 +120,8 @@ class BlockStreamManagerImplTest { private static final BlockItem FAKE_STATE_CHANGES = BlockItem.newBuilder() .stateChanges(StateChanges.newBuilder().consensusTimestamp(CONSENSUS_THEN)) .build(); - // TODO: remove, or replace with wrapped record file item - private static final BlockItem FAKE_RECORD_FILE_ITEM = null; + private static final BlockItem FAKE_RECORD_FILE_ITEM = + BlockItem.newBuilder().recordFile(RecordFileItem.DEFAULT).build(); private final InitialStateHash hashInfo = new InitialStateHash(completedFuture(ZERO_BLOCK_HASH), 0); @Mock @@ -154,9 +162,6 @@ class BlockStreamManagerImplTest { @Mock private HederaVirtualMapState state; - @Mock - private Iterator mockIterator; - @Mock private ConsensusEvent mockEvent; @@ -233,24 +238,23 @@ void classifiesNonGenesisBlockOfSameVersionWithWorkDoneAsNoWork() { @Test void canUpdateDistinguishedTimes() { given(configProvider.getConfiguration()).willReturn(new VersionedConfigImpl(DEFAULT_CONFIG, 1L)); - // TODO: fix - // subject = new BlockStreamManagerImpl( - // blockHashSigner, - // () -> aWriter, - // ForkJoinPool.commonPool(), - // configProvider, - // networkInfo, - // boundaryStateChangeListener, - // hashInfo, - // SemanticVersion.DEFAULT, - // TEST_PLATFORM_STATE_FACADE, - // lifecycle, - // metrics); - assertSame(Instant.EPOCH, subject.lastIntervalProcessTime()); + subject = new BlockStreamManagerImpl( + blockHashSigner, + () -> aWriter, + ForkJoinPool.commonPool(), + configProvider, + networkInfo, + boundaryStateChangeListener, + hashInfo, + SemanticVersion.DEFAULT, + TEST_PLATFORM_STATE_FACADE, + lifecycle, + metrics); + assertSame(EPOCH, subject.lastIntervalProcessTime()); subject.setLastIntervalProcessTime(CONSENSUS_NOW); assertEquals(CONSENSUS_NOW, subject.lastIntervalProcessTime()); - assertSame(Instant.EPOCH, subject.lastTopLevelConsensusTime()); + assertSame(EPOCH, subject.lastTopLevelConsensusTime()); subject.setLastTopLevelTime(CONSENSUS_NOW); assertEquals(CONSENSUS_NOW, subject.lastTopLevelConsensusTime()); } @@ -258,19 +262,18 @@ void canUpdateDistinguishedTimes() { @Test void requiresLastHashToBeInitialized() { given(configProvider.getConfiguration()).willReturn(new VersionedConfigImpl(DEFAULT_CONFIG, 1)); - // TODO: fix - // subject = new BlockStreamManagerImpl( - // blockHashSigner, - // () -> aWriter, - // ForkJoinPool.commonPool(), - // configProvider, - // networkInfo, - // boundaryStateChangeListener, - // hashInfo, - // SemanticVersion.DEFAULT, - // TEST_PLATFORM_STATE_FACADE, - // lifecycle, - // metrics); + subject = new BlockStreamManagerImpl( + blockHashSigner, + () -> aWriter, + ForkJoinPool.commonPool(), + configProvider, + networkInfo, + boundaryStateChangeListener, + hashInfo, + SemanticVersion.DEFAULT, + TEST_PLATFORM_STATE_FACADE, + lifecycle, + metrics); assertThrows(IllegalStateException.class, () -> subject.startRound(round, state)); } @@ -288,7 +291,7 @@ void startsAndEndsBlockWithSingleRoundPerBlockAsExpected() throws ParseException given(round.getRoundNum()).willReturn(ROUND_NO); // Initialize the last (N-1) block hash - subject.initLastBlockHash(FAKE_RESTART_BLOCK_HASH); + subject.init(state, FAKE_RESTART_BLOCK_HASH); assertFalse(subject.hasLedgerId()); given(blockHashSigner.isReady()).willReturn(true); @@ -327,40 +330,38 @@ void startsAndEndsBlockWithSingleRoundPerBlockAsExpected() throws ParseException verify(aWriter).openBlock(N_BLOCK_NO); - // TODO: Assert the internal state of the subject has changed as expected and the writer has been closed - // final var expectedBlockInfo = new BlockStreamInfo( - // N_BLOCK_NO, - // asTimestamp(CONSENSUS_NOW), - // appendHash(combine(ZERO_BLOCK_HASH, FAKE_RESULT_HASH), appendHash(ZERO_BLOCK_HASH, - // Bytes.EMPTY, 4), 4), - // appendHash(FAKE_RESTART_BLOCK_HASH, appendHash(N_MINUS_2_BLOCK_HASH, Bytes.EMPTY, 256), 256), - // Bytes.fromHex( - // - // "edde6b2beddb2fda438665bbe6df0a639c518e6d5352e7276944b70777d437d28d1b22813ed70f5b8a3a3cbaf08aa9a8"), - // ZERO_BLOCK_HASH, - // 2, - // List.of( - // Bytes.EMPTY, - // Bytes.fromHex( - // - // "839ddb854c8f4cf9c3705268b17bc7d53e91454ff14dbbfffd6c77b6118a0e79fb1e478b4924bfb0fd93ef60101d3237")), - // FAKE_TRANSACTION_RESULT.transactionResultOrThrow().consensusTimestampOrThrow(), - // true, - // SemanticVersion.DEFAULT, - // CONSENSUS_THEN, - // CONSENSUS_THEN, - // Bytes.fromHex( - // - // "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), - // Bytes.fromHex( - // - // "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), - // Bytes.fromHex( - // - // "bf99e1dfd15ffe551ae4bc0953f396639755f0419522f323875806a55a57dca6a4df61ea6dee28bec0c37ed54881d392")); - // - // final var actualBlockInfo = infoRef.get(); - // assertEquals(expectedBlockInfo, actualBlockInfo); + // Assert the internal state of the subject has changed as expected and the writer has been closed + final var expectedBlockInfo = new BlockStreamInfo( + N_BLOCK_NO, + asTimestamp(CONSENSUS_NOW), + appendHash(combine(ZERO_BLOCK_HASH, FAKE_RESULT_HASH), appendHash(ZERO_BLOCK_HASH, Bytes.EMPTY, 4), 4), + appendHash(FAKE_RESTART_BLOCK_HASH, appendHash(N_MINUS_2_BLOCK_HASH, Bytes.EMPTY, 256), 256), + Bytes.fromHex( + "edde6b2beddb2fda438665bbe6df0a639c518e6d5352e7276944b70777d437d28d1b22813ed70f5b8a3a3cbaf08aa9a8"), + ZERO_BLOCK_HASH, + 2, + List.of( + Bytes.EMPTY, + Bytes.fromHex( + "839ddb854c8f4cf9c3705268b17bc7d53e91454ff14dbbfffd6c77b6118a0e79fb1e478b4924bfb0fd93ef60101d3237")), + FAKE_TRANSACTION_RESULT.transactionResultOrThrow().consensusTimestampOrThrow(), + true, + SemanticVersion.DEFAULT, + CONSENSUS_THEN, + CONSENSUS_THEN, + Bytes.fromHex( + "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), + Bytes.fromHex( + "bf99e1dfd15ffe551ae4bc0953f396639755f0419522f323875806a55a57dca6a4df61ea6dee28bec0c37ed54881d392"), + Bytes.fromHex( + "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), + List.of( + Bytes.fromHex( + "a63602dae8cc657abca1999f948de14320ab2c48d58994f14abce574607d859e35acf7cb2305be511a3099243ccd876d")), + 1); + + final var actualBlockInfo = infoRef.get(); + assertEquals(expectedBlockInfo, actualBlockInfo); // Assert the block proof was written final var proofItem = lastAItem.get(); @@ -369,8 +370,7 @@ void startsAndEndsBlockWithSingleRoundPerBlockAsExpected() throws ParseException assertTrue(item.hasBlockProof()); final var proof = item.blockProofOrThrow(); assertEquals(N_BLOCK_NO, proof.block()); - // TODO: restore - // assertEquals(FIRST_FAKE_SIGNATURE, proof.blockSignature()); + assertEquals(FIRST_FAKE_SIGNATURE, proof.signedBlockProof().blockSignature()); } @Test @@ -427,7 +427,7 @@ void blockWithNoUserTransactionsHasExpectedHeader() { given(round.getRoundNum()).willReturn(ROUND_NO); // Initialize the last (N-1) block hash - subject.initLastBlockHash(FAKE_RESTART_BLOCK_HASH); + subject.init(state, FAKE_RESTART_BLOCK_HASH); assertFalse(subject.hasLedgerId()); given(blockHashSigner.isReady()).willReturn(true); @@ -527,7 +527,7 @@ void alwaysEndsBlockOnFreezeRoundPerBlockAsExpected() throws ParseException { given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); // Initialize the last (N-1) block hash - subject.initLastBlockHash(FAKE_RESTART_BLOCK_HASH); + subject.init(state, FAKE_RESTART_BLOCK_HASH); given(blockHashSigner.isReady()).willReturn(true); // Start the round that will be block N @@ -562,38 +562,37 @@ void alwaysEndsBlockOnFreezeRoundPerBlockAsExpected() throws ParseException { verify(aWriter).openBlock(N_BLOCK_NO); - // TODO: Assert the internal state of the subject has changed as expected and the writer has been closed - // final var expectedBlockInfo = new BlockStreamInfo( - // N_BLOCK_NO, - // asTimestamp(CONSENSUS_NOW), - // appendHash(combine(Bytes.fromHex("dd".repeat(48)), FAKE_RESULT_HASH), resultHashes, 4), - // appendHash(FAKE_RESTART_BLOCK_HASH, appendHash(N_MINUS_2_BLOCK_HASH, Bytes.EMPTY, 256), 256), - // Bytes.fromHex( - // - // "edde6b2beddb2fda438665bbe6df0a639c518e6d5352e7276944b70777d437d28d1b22813ed70f5b8a3a3cbaf08aa9a8"), - // ZERO_BLOCK_HASH, - // 2, - // List.of( - // Bytes.EMPTY, - // Bytes.fromHex( - // - // "839ddb854c8f4cf9c3705268b17bc7d53e91454ff14dbbfffd6c77b6118a0e79fb1e478b4924bfb0fd93ef60101d3237")), - // FAKE_TRANSACTION_RESULT.transactionResultOrThrow().consensusTimestampOrThrow(), - // false, - // SemanticVersion.DEFAULT, - // CONSENSUS_THEN, - // CONSENSUS_THEN, - // Bytes.fromHex( - // - // "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), - // Bytes.fromHex( - // - // "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), - // Bytes.fromHex( - // - // "8ee0718d5f75f867f85cb4e400ebf7bfbb4cd91479d7f3f8bfd28ce062c318c312b8f4de185a994b78337e6391e3f000")); - // final var actualBlockInfo = infoRef.get(); - // assertEquals(expectedBlockInfo, actualBlockInfo); + // Assert the internal state of the subject has changed as expected and the writer has been closed + final var expectedBlockInfo = new BlockStreamInfo( + N_BLOCK_NO, + asTimestamp(CONSENSUS_NOW), + appendHash(combine(Bytes.fromHex("dd".repeat(48)), FAKE_RESULT_HASH), resultHashes, 4), + appendHash(FAKE_RESTART_BLOCK_HASH, appendHash(N_MINUS_2_BLOCK_HASH, Bytes.EMPTY, 256), 256), + Bytes.fromHex( + "edde6b2beddb2fda438665bbe6df0a639c518e6d5352e7276944b70777d437d28d1b22813ed70f5b8a3a3cbaf08aa9a8"), + ZERO_BLOCK_HASH, + 2, + List.of( + Bytes.EMPTY, + Bytes.fromHex( + "839ddb854c8f4cf9c3705268b17bc7d53e91454ff14dbbfffd6c77b6118a0e79fb1e478b4924bfb0fd93ef60101d3237")), + FAKE_TRANSACTION_RESULT.transactionResultOrThrow().consensusTimestampOrThrow(), + false, + SemanticVersion.DEFAULT, + CONSENSUS_THEN, + CONSENSUS_THEN, + Bytes.fromHex( + "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), + Bytes.fromHex( + "8ee0718d5f75f867f85cb4e400ebf7bfbb4cd91479d7f3f8bfd28ce062c318c312b8f4de185a994b78337e6391e3f000"), + Bytes.fromHex( + "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"), + List.of( + Bytes.fromHex( + "a63602dae8cc657abca1999f948de14320ab2c48d58994f14abce574607d859e35acf7cb2305be511a3099243ccd876d")), + 1); + final var actualBlockInfo = infoRef.get(); + assertEquals(expectedBlockInfo, actualBlockInfo); // Assert the block proof was written final var proofItem = lastAItem.get(); @@ -602,8 +601,7 @@ void alwaysEndsBlockOnFreezeRoundPerBlockAsExpected() throws ParseException { assertTrue(item.hasBlockProof()); final var proof = item.blockProofOrThrow(); assertEquals(N_BLOCK_NO, proof.block()); - // TODO: restore - // assertEquals(FIRST_FAKE_SIGNATURE, proof.blockSignature()); + assertEquals(FIRST_FAKE_SIGNATURE, proof.signedBlockProof().blockSignature()); } @Test @@ -628,7 +626,7 @@ void supportsMultiplePendingBlocksWithIndirectProofAsExpected() throws ParseExce given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); // Initialize the last (N-1) block hash - subject.initLastBlockHash(FAKE_RESTART_BLOCK_HASH); + subject.init(state, FAKE_RESTART_BLOCK_HASH); // Start the round that will be block N subject.startRound(round, state); @@ -675,9 +673,13 @@ void supportsMultiplePendingBlocksWithIndirectProofAsExpected() throws ParseExce assertTrue(aItem.hasBlockProof()); final var aProof = aItem.blockProofOrThrow(); assertEquals(N_BLOCK_NO, aProof.block()); - // TODO: restore - // assertEquals(FIRST_FAKE_SIGNATURE, aProof.blockSignature()); - assertEquals(3, aProof.siblingHashes().size()); + assertEquals( + FIRST_FAKE_SIGNATURE, + aProof.blockStateProof().signedBlockProof().blockSignature()); + // Since the state proof's first merkle path should be the leaf containing the block merkle tree's previous + // block hash (4 levels descended from the signed block root hash), there are four required siblings to keep for + // indirect proofs + assertEquals(4, aProof.siblingHashes().size()); // And the proof for N+1 using a direct proof final var bProofItem = lastBItem.get(); assertNotNull(bProofItem); @@ -685,8 +687,7 @@ void supportsMultiplePendingBlocksWithIndirectProofAsExpected() throws ParseExce assertTrue(bItem.hasBlockProof()); final var bProof = bItem.blockProofOrThrow(); assertEquals(N_BLOCK_NO + 1, bProof.block()); - // TODO: restore - // assertEquals(FIRST_FAKE_SIGNATURE, bProof.blockSignature()); + assertEquals(FIRST_FAKE_SIGNATURE, bProof.signedBlockProof().blockSignature()); assertTrue(bProof.siblingHashes().isEmpty()); verify(indirectProofsCounter).increment(); @@ -717,21 +718,29 @@ void createsBlockWhenTimePeriodElapses() { .thenAcceptAsync(any()); // When starting a round at t=0 - given(round.getConsensusTimestamp()).willReturn(Instant.ofEpochSecond(1000)); - subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); + var time = EPOCH; + mockRoundWithTxnTimestamp(time); + subject.init(state, N_MINUS_2_BLOCK_HASH); subject.startRound(round, state); + subject.endRound(state, ROUND_NO); // And another round at t=1 - given(round.getConsensusTimestamp()).willReturn(Instant.ofEpochSecond(1001)); + time = EPOCH.plusSeconds(1); + mockRoundWithTxnTimestamp(time); subject.startRound(round, state); + // Advance tracked block end timestamp to t=1 + subject.writeItem(transactionResultItemFrom(time)); subject.endRound(state, ROUND_NO); // Then block should not be closed verify(aWriter, never()).closeCompleteBlock(); // When starting another round at t=3 (after period) - given(round.getConsensusTimestamp()).willReturn(Instant.ofEpochSecond(1003)); + time = EPOCH.plusSeconds(3); + mockRoundWithTxnTimestamp(time); subject.startRound(round, state); + // Advance tracked block end timestamp to t=3 + subject.writeItem(transactionResultItemFrom(time)); subject.endRound(state, ROUND_NO); // Then block should be closed @@ -749,17 +758,20 @@ void doesNotCreateBlockWhenTimePeriodNotElapsed() { platformStateWithFreezeTime(null), aWriter); givenEndOfRoundSetup(); - given(round.getRoundNum()).willReturn(ROUND_NO); given(blockHashSigner.isReady()).willReturn(true); // When starting a round at t=0 - given(round.getConsensusTimestamp()).willReturn(Instant.ofEpochSecond(1000)); + var time = EPOCH; + mockRoundWithTxnTimestamp(time); subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); subject.startRound(round, state); // And another round at t=1.5 - given(round.getConsensusTimestamp()).willReturn(Instant.ofEpochSecond(1001, 500_000_000)); + time = EPOCH.plusSeconds(1).plusNanos(500_000_000); + mockRoundWithTxnTimestamp(time); subject.startRound(round, state); + // Advance tracked block end timestamp to t=1.5 + subject.writeItem(transactionResultItemFrom(time)); subject.endRound(state, ROUND_NO); // Then block should not be closed @@ -796,13 +808,17 @@ void alwaysEndsBlockOnFreezeRoundEvenIfPeriodNotElapsed() { .thenAcceptAsync(any()); // When starting a round at t=0 - given(round.getConsensusTimestamp()).willReturn(Instant.ofEpochSecond(1000)); - subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); + var time = EPOCH; + mockRoundWithTxnTimestamp(time); + subject.init(state, N_MINUS_2_BLOCK_HASH); subject.startRound(round, state); // And another round at t=1 with freeze - given(round.getConsensusTimestamp()).willReturn(Instant.ofEpochSecond(1001)); + time = EPOCH.plusSeconds(1); + mockRoundWithTxnTimestamp(time); subject.startRound(round, state); + // Advance tracked block end timestamp to t=1 + subject.writeItem(transactionResultItemFrom(time)); subject.endRound(state, ROUND_NO); // Then block should be closed due to freeze, even though period not elapsed @@ -834,7 +850,7 @@ void usesRoundsPerBlockWhenBlockPeriodIsZero() { // When processing rounds given(round.getConsensusTimestamp()).willReturn(Instant.ofEpochSecond(1000)); - subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); + subject.init(state, N_MINUS_2_BLOCK_HASH); // First round (not mod 2) given(round.getRoundNum()).willReturn(3L); @@ -907,7 +923,7 @@ void eventHashMapIsClearedBetweenBlocks() { .thenAcceptAsync(any()); // Initialize hash and start a round - subject.initLastBlockHash(FAKE_RESTART_BLOCK_HASH); + subject.init(state, FAKE_RESTART_BLOCK_HASH); subject.startRound(round, state); // Track event hashes in the first block @@ -942,7 +958,6 @@ void eventHashMapIsClearedBetweenBlocks() { } @Test - @SuppressWarnings("unchecked") void writesBlockFooterBeforeBlockProof() { // Given a manager with a single round per block givenSubjectWith( @@ -967,52 +982,49 @@ void writesBlockFooterBeforeBlockProof() { given(round.getRoundNum()).willReturn(ROUND_NO); given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); given(blockHashSigner.isReady()).willReturn(true); - // TODO: fix - // given(blockHashSigner.schemeId()).willReturn(1L); - // - // // Set up the signature future to complete immediately - // given(blockHashSigner.signFuture(any())).willReturn(mockSigningFuture); - // doAnswer(invocationOnMock -> { - // final Consumer consumer = invocationOnMock.getArgument(0); - // consumer.accept(FIRST_FAKE_SIGNATURE); - // return null; - // }) - // .when(mockSigningFuture) - // .thenAcceptAsync(any()); - // - // // Initialize hash and start a round - // subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); - // subject.startRound(round, state); - // - // // Write some items - // subject.writeItem(FAKE_SIGNED_TRANSACTION); - // subject.writeItem(FAKE_TRANSACTION_RESULT); - // subject.writeItem(FAKE_STATE_CHANGES); - // - // // End the round - // subject.endRound(state, ROUND_NO); - // - // // Verify BlockFooter was written - // assertNotNull(footerItem.get(), "BlockFooter should be written"); - // assertTrue(footerItem.get().hasBlockFooter()); - // - // final var footer = footerItem.get().blockFooterOrThrow(); - // assertNotNull(footer.previousBlockRootHash(), "Previous block root hash should be set"); - // // TODO(#21210): Currently using NULL_HASH placeholder for block hashes tree - // // Will be replaced when streaming merkle tree of all block hashes is implemented - // assertEquals( - // BlockStreamManagerImpl.NULL_HASH, - // footer.rootHashOfAllBlockHashesTree(), - // "Block hashes tree root should be NULL_HASH until #21210 is implemented"); - // assertNotNull(footer.startOfBlockStateRootHash(), "Start of block state root hash should be set"); - // - // // Verify BlockProof was also written - // assertNotNull(proofItem.get(), "BlockProof should be written"); - // assertTrue(proofItem.get().hasBlockProof()); + + // Set up the signature future to complete immediately + given(blockHashSigner.sign(any())) + .willReturn(new BlockHashSigner.Attempt(Bytes.EMPTY, ChainOfTrustProof.DEFAULT, mockSigningFuture)); + doAnswer(invocationOnMock -> { + final Consumer consumer = invocationOnMock.getArgument(0); + consumer.accept(FIRST_FAKE_SIGNATURE); + return null; + }) + .when(mockSigningFuture) + .thenAcceptAsync(any()); + + // Initialize hash and start a round + subject.init(state, N_MINUS_2_BLOCK_HASH); + subject.startRound(round, state); + + // Write some items + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.writeItem(FAKE_TRANSACTION_RESULT); + subject.writeItem(FAKE_STATE_CHANGES); + + // End the round + subject.endRound(state, ROUND_NO); + + // Verify BlockFooter was written + assertNotNull(footerItem.get(), "BlockFooter should be written"); + assertTrue(footerItem.get().hasBlockFooter()); + + final var footer = footerItem.get().blockFooterOrThrow(); + assertNotNull(footer.previousBlockRootHash(), "Previous block root hash should be set"); + assertEquals( + Bytes.fromHex( + "bf95370fb03d71634b937b1a19b4f8c445914600247af794788a322b1c4123798e94e328b34f166b38a9a6b1aea93414"), + footer.rootHashOfAllBlockHashesTree(), + "Block hashes tree root should be NULL_HASH until #21210 is implemented"); + assertNotNull(footer.startOfBlockStateRootHash(), "Start of block state root hash should be set"); + + // Verify BlockProof was also written + assertNotNull(proofItem.get(), "BlockProof should be written"); + assertTrue(proofItem.get().hasBlockProof()); } @Test - @SuppressWarnings("unchecked") void blockFooterContainsCorrectHashValues() { // Given a manager with a single round per block givenSubjectWith( @@ -1034,46 +1046,46 @@ void blockFooterContainsCorrectHashValues() { given(round.getRoundNum()).willReturn(ROUND_NO); given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); given(blockHashSigner.isReady()).willReturn(true); - // TODO: fix - // given(blockHashSigner.schemeId()).willReturn(1L); - // - // // Set up the signature future - // given(blockHashSigner.signFuture(any())).willReturn(mockSigningFuture); - // doAnswer(invocationOnMock -> { - // final Consumer consumer = invocationOnMock.getArgument(0); - // consumer.accept(FIRST_FAKE_SIGNATURE); - // return null; - // }) - // .when(mockSigningFuture) - // .thenAcceptAsync(any()); - // - // // Initialize with known hash and start round - // subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); - // subject.startRound(round, state); - // subject.writeItem(FAKE_SIGNED_TRANSACTION); - // subject.endRound(state, ROUND_NO); - // - // // Verify BlockFooter hash values - // assertNotNull(footerItem.get(), "BlockFooter should be written"); - // final var footer = footerItem.get().blockFooterOrThrow(); - // - // // Verify previousBlockRootHash matches the last block hash - // assertEquals( - // N_MINUS_2_BLOCK_HASH, - // footer.previousBlockRootHash(), - // "Previous block root hash should match initialized last block hash"); - // - // // Verify rootHashOfAllBlockHashesTree is NULL_HASH (placeholder) - // assertEquals( - // BlockStreamManagerImpl.NULL_HASH, - // footer.rootHashOfAllBlockHashesTree(), - // "Block hashes tree root should be NULL_HASH placeholder"); - // - // // Verify startOfBlockStateRootHash is set - // assertEquals( - // FAKE_START_OF_BLOCK_STATE_HASH.getBytes(), - // footer.startOfBlockStateRootHash(), - // "Start of block state root hash should match expected value"); + + // Set up the signature future + given(blockHashSigner.sign(any())) + .willReturn(new BlockHashSigner.Attempt(Bytes.EMPTY, ChainOfTrustProof.DEFAULT, mockSigningFuture)); + doAnswer(invocationOnMock -> { + final Consumer consumer = invocationOnMock.getArgument(0); + consumer.accept(FIRST_FAKE_SIGNATURE); + return null; + }) + .when(mockSigningFuture) + .thenAcceptAsync(any()); + + // Initialize with known hash and start round + subject.init(state, N_MINUS_2_BLOCK_HASH); + subject.startRound(round, state); + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.endRound(state, ROUND_NO); + + // Verify BlockFooter hash values + assertNotNull(footerItem.get(), "BlockFooter should be written"); + final var footer = footerItem.get().blockFooterOrThrow(); + + // Verify previousBlockRootHash matches the last block hash + assertEquals( + N_MINUS_2_BLOCK_HASH, + footer.previousBlockRootHash(), + "Previous block root hash should match initialized last block hash"); + + // Verify rootHashOfAllBlockHashesTree is correct + assertEquals( + Bytes.fromHex( + "bf95370fb03d71634b937b1a19b4f8c445914600247af794788a322b1c4123798e94e328b34f166b38a9a6b1aea93414"), + footer.rootHashOfAllBlockHashesTree(), + "Block hashes tree root should be NULL_HASH placeholder"); + + // Verify startOfBlockStateRootHash is set + assertEquals( + FAKE_START_OF_BLOCK_STATE_HASH.getBytes(), + footer.startOfBlockStateRootHash(), + "Start of block state root hash should match expected value"); } @Test @@ -1114,38 +1126,36 @@ void blockFooterWrittenForEachBlock() { given(round.getRoundNum()).willReturn(ROUND_NO); given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW); given(blockHashSigner.isReady()).willReturn(true); - // TODO: fix - // given(blockHashSigner.schemeId()).willReturn(1L); - // - // // Set up the signature futures - // final CompletableFuture firstSignature = (CompletableFuture) - // mock(CompletableFuture.class); - // final CompletableFuture secondSignature = (CompletableFuture) - // mock(CompletableFuture.class); - // given(blockHashSigner.signFuture(any())).willReturn(firstSignature).willReturn(secondSignature); - // - // // Initialize and create first block - // subject.initLastBlockHash(FAKE_RESTART_BLOCK_HASH); - // subject.startRound(round, state); - // subject.writeItem(FAKE_SIGNED_TRANSACTION); - // subject.endRound(state, ROUND_NO); - // - // // Create second block - // given(round.getRoundNum()).willReturn(ROUND_NO + 1); - // given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW.plusSeconds(1)); - // given(notification.round()).willReturn(ROUND_NO); - // given(notification.hash()).willReturn(FAKE_START_OF_BLOCK_STATE_HASH); - // subject.notify(notification); - // subject.startRound(round, state); - // subject.writeItem(FAKE_SIGNED_TRANSACTION); - // subject.endRound(state, ROUND_NO + 1); - // - // // Verify BlockFooter was written for each block - // assertEquals(2, footerItems.size(), "Should have written BlockFooter for each block"); - // - // // Verify both are valid BlockFooters - // assertTrue(footerItems.get(0).hasBlockFooter(), "First item should be BlockFooter"); - // assertTrue(footerItems.get(1).hasBlockFooter(), "Second item should be BlockFooter"); + + // Set up the signature futures + final CompletableFuture firstSignature = (CompletableFuture) mock(CompletableFuture.class); + final CompletableFuture secondSignature = (CompletableFuture) mock(CompletableFuture.class); + given(blockHashSigner.sign(any())) + .willReturn(new BlockHashSigner.Attempt(Bytes.EMPTY, ChainOfTrustProof.DEFAULT, firstSignature)) + .willReturn(new BlockHashSigner.Attempt(Bytes.EMPTY, ChainOfTrustProof.DEFAULT, secondSignature)); + + // Initialize and create first block + subject.init(state, FAKE_RESTART_BLOCK_HASH); + subject.startRound(round, state); + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.endRound(state, ROUND_NO); + + // Create second block + given(round.getRoundNum()).willReturn(ROUND_NO + 1); + given(round.getConsensusTimestamp()).willReturn(CONSENSUS_NOW.plusSeconds(1)); + given(notification.round()).willReturn(ROUND_NO); + given(notification.hash()).willReturn(FAKE_START_OF_BLOCK_STATE_HASH); + subject.notify(notification); + subject.startRound(round, state); + subject.writeItem(FAKE_SIGNED_TRANSACTION); + subject.endRound(state, ROUND_NO + 1); + + // Verify BlockFooter was written for each block + assertEquals(2, footerItems.size(), "Should have written BlockFooter for each block"); + + // Verify both are valid BlockFooters + assertTrue(footerItems.get(0).hasBlockFooter(), "First item should be BlockFooter"); + assertTrue(footerItems.get(1).hasBlockFooter(), "Second item should be BlockFooter"); } @Test @@ -1194,19 +1204,18 @@ private void givenSubjectWith( .withValue("blockStream.blockPeriod", Duration.of(blockPeriod, ChronoUnit.SECONDS)) .getOrCreateConfig(); given(configProvider.getConfiguration()).willReturn(new VersionedConfigImpl(config, 1L)); - // TODO: fix - // subject = new BlockStreamManagerImpl( - // blockHashSigner, - // () -> writers[nextWriter.getAndIncrement()], - // ForkJoinPool.commonPool(), - // configProvider, - // networkInfo, - // boundaryStateChangeListener, - // hashInfo, - // SemanticVersion.DEFAULT, - // TEST_PLATFORM_STATE_FACADE, - // lifecycle, - // metrics); + subject = new BlockStreamManagerImpl( + blockHashSigner, + () -> writers[nextWriter.getAndIncrement()], + ForkJoinPool.commonPool(), + configProvider, + networkInfo, + boundaryStateChangeListener, + hashInfo, + SemanticVersion.DEFAULT, + TEST_PLATFORM_STATE_FACADE, + lifecycle, + metrics); given(state.getReadableStates(any())).willReturn(readableStates); given(readableStates.getSingleton(PLATFORM_STATE_STATE_ID)).willReturn(platformStateReadableSingletonState); lenient().when(state.getReadableStates(FreezeServiceImpl.NAME)).thenReturn(readableStates); @@ -1222,9 +1231,7 @@ private void givenEndOfRoundSetup() { private void givenEndOfRoundSetup(@Nullable final AtomicReference headerRef) { // Add mock for round iterator - lenient().when(round.iterator()).thenReturn(mockIterator); - lenient().when(mockIterator.next()).thenReturn(mockEvent); - lenient().when(mockEvent.getConsensusTimestamp()).thenReturn(CONSENSUS_NOW); + mockRoundWithTxnTimestamp(CONSENSUS_NOW); lenient() .doAnswer(invocationOnMock -> { lastAItem.set(invocationOnMock.getArgument(1)); @@ -1280,7 +1287,29 @@ private PlatformState platformStateWithFreezeTime(@Nullable final Instant freeze .build(); } + private void mockRound(Instant timestamp) { + given(round.getRoundNum()).willReturn(ROUND_NO); + lenient().when(round.iterator()).thenReturn(new Arrays.Iterator<>(new ConsensusEvent[] {mockEvent})); + lenient().when(round.getConsensusTimestamp()).thenReturn(timestamp); + } + private static Bytes noThrowSha384HashOfItem(@NonNull final BlockItem item) { return Bytes.wrap(noThrowSha384HashOf(BlockItem.PROTOBUF.toBytes(item).toByteArray())); } + + private void mockRoundWithTxnTimestamp(Instant timestamp) { + mockRound(timestamp); + + final var txn = new TransactionWrapper(Bytes.fromHex("abcdefABCDEF")); + txn.setConsensusTimestamp(timestamp); + lenient() + .when(mockEvent.consensusTransactionIterator()) + .thenReturn(new Arrays.Iterator<>(new ConsensusTransaction[] {txn})); + } + + private BlockItem transactionResultItemFrom(Instant consensusTimestamp) { + return BlockItem.newBuilder() + .transactionResult(TransactionResult.newBuilder().consensusTimestamp(asTimestamp(consensusTimestamp))) + .build(); + } } From 7badf1716a191d9825688b8c55b32e7710364a80 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Sun, 2 Nov 2025 23:30:29 -0700 Subject: [PATCH 57/63] One more test Signed-off-by: Matt Hess --- .../hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index 8099051cd02c..4a456efe237f 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -1182,7 +1182,7 @@ void blockFooterNotWrittenWhenBlockNotClosed() { given(blockHashSigner.isReady()).willReturn(true); // Initialize and start first round (block not yet closed) - subject.initLastBlockHash(N_MINUS_2_BLOCK_HASH); + subject.init(state, N_MINUS_2_BLOCK_HASH); subject.startRound(round, state); subject.writeItem(FAKE_SIGNED_TRANSACTION); subject.endRound(state, ROUND_NO); From 7c1e2145cb68224da9e3b3da071261b2781ae7a8 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 3 Nov 2025 09:11:45 -0700 Subject: [PATCH 58/63] Re-enable tests, remove old comments Signed-off-by: Matt Hess --- .../app/blocks/BlockStreamBuilderTest.java | 44 +++++++++---------- .../blocks/impl/streaming/BlockTestUtils.java | 2 - .../streaming/GrpcBlockItemWriterTest.java | 5 +-- .../schemas/V0560BlockStreamSchemaTest.java | 40 +++++++++-------- 4 files changed, 43 insertions(+), 48 deletions(-) diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java index d6b6e120440e..0c70bca01cc9 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/BlockStreamBuilderTest.java @@ -10,6 +10,7 @@ import static com.hedera.node.app.spi.workflows.HandleContext.TransactionCategory.USER; import static com.hedera.node.app.spi.workflows.record.StreamBuilder.ReversingBehavior.REVERSIBLE; import static com.hedera.node.app.spi.workflows.record.StreamBuilder.SignedTxCustomizer.NOOP_SIGNED_TX_CUSTOMIZER; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -140,12 +141,11 @@ void testBlockItemsWithTraceAndOutput() { assertTrue(output.hasContractCall()); final var traceItem = blockItems.get(3); - // TODO: assert trace data - // assertTrue(traceItem.hasTraceData()); - // final var trace = traceItem.traceDataOrThrow(); - // assertTrue(trace.hasEvmTraceData()); - // final var evmTrace = trace.evmTraceDataOrThrow(); - // assertEquals(usages, evmTrace.contractSlotUsages()); + assertTrue(traceItem.hasTraceData()); + final var trace = traceItem.traceDataOrThrow(); + assertTrue(trace.hasEvmTraceData()); + final var evmTrace = trace.evmTraceDataOrThrow(); + assertEquals(usages, evmTrace.contractSlotUsages()); } @Test @@ -161,15 +161,14 @@ void testBlockItemsWithAdditionalAutomaticTokenAssociationTraceData() { final var blockItems = itemsBuilder.build(false, List.of()).blockItems(); final var traceItem = blockItems.get(2); - // TODO: assert trace data - // assertThat(traceItem.hasTraceData()).isTrue(); - // final var trace = traceItem.traceDataOrThrow(); - // - // assertThat(trace.hasAutoAssociateTraceData()).isTrue(); - // final var autoAssociateTraceData = trace.autoAssociateTraceData(); - // assertThat(autoAssociateTraceData).isNotNull(); - // assertThat(autoAssociateTraceData.automaticTokenAssociations().accountNum()) - // .isEqualTo(2); + assertThat(traceItem.hasTraceData()).isTrue(); + final var trace = traceItem.traceDataOrThrow(); + + assertThat(trace.hasAutoAssociateTraceData()).isTrue(); + final var autoAssociateTraceData = trace.autoAssociateTraceData(); + assertThat(autoAssociateTraceData).isNotNull(); + assertThat(autoAssociateTraceData.automaticTokenAssociations().accountNum()) + .isEqualTo(2); } @Test @@ -180,14 +179,13 @@ void testBlockItemsWithAdditionalSubmitMsgTraceData() { final var blockItems = itemsBuilder.build(false, List.of()).blockItems(); final var traceItem = blockItems.get(2); - // TODO: assert trace data - // assertThat(traceItem.hasTraceData()).isTrue(); - // final var trace = traceItem.traceDataOrThrow(); - // - // assertThat(trace.hasSubmitMessageTraceData()).isTrue(); - // final var submitMessageTraceData = trace.submitMessageTraceData(); - // assertThat(submitMessageTraceData).isNotNull(); - // assertThat(submitMessageTraceData.sequenceNumber()).isEqualTo(66); + assertThat(traceItem.hasTraceData()).isTrue(); + final var trace = traceItem.traceDataOrThrow(); + + assertThat(trace.hasSubmitMessageTraceData()).isTrue(); + final var submitMessageTraceData = trace.submitMessageTraceData(); + assertThat(submitMessageTraceData).isNotNull(); + assertThat(submitMessageTraceData.sequenceNumber()).isEqualTo(66); } @Test diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockTestUtils.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockTestUtils.java index 0dd37ce2bf7c..863cabe49e89 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockTestUtils.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockTestUtils.java @@ -223,8 +223,6 @@ public static BlockItem newBlockProof(final long blockNumber) { final BlockProof proof = BlockProof.newBuilder() .block(blockNumber) .verificationKey(VERIFICATION_KEY) - // TODO: add TssSigned or StateProof (includes sig) - // ???what about history proof? .build(); return BlockItem.newBuilder().blockProof(proof).build(); } diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/GrpcBlockItemWriterTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/GrpcBlockItemWriterTest.java index c327f820bae2..e5bed6b4db77 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/GrpcBlockItemWriterTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/GrpcBlockItemWriterTest.java @@ -57,10 +57,7 @@ void testWritePbjItemAndBytes() { // Create BlockProof as easiest way to build object from BlockStreams Bytes bytes = Bytes.wrap(new byte[] {1, 2, 3, 4, 5}); final var proof = BlockItem.newBuilder() - .blockProof(BlockProof.newBuilder() - // TODO: add TssSigned or StateProof (includes sig) - // .blockSignature(bytes) - .siblingHashes(new ArrayList<>())) + .blockProof(BlockProof.newBuilder().siblingHashes(new ArrayList<>())) .build(); grpcBlockItemWriter.writePbjItemAndBytes(proof, bytes); diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/schemas/V0560BlockStreamSchemaTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/schemas/V0560BlockStreamSchemaTest.java index 3444d94fa2fe..4bc1fcd417a4 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/schemas/V0560BlockStreamSchemaTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/schemas/V0560BlockStreamSchemaTest.java @@ -19,6 +19,7 @@ import com.swirlds.state.lifecycle.MigrationContext; import com.swirlds.state.spi.WritableSingletonState; import com.swirlds.state.spi.WritableStates; +import java.util.List; import java.util.Map; import java.util.function.Consumer; import org.junit.jupiter.api.BeforeEach; @@ -105,25 +106,26 @@ void assumesMigrationIfNotGenesisAndStateIsNull() { subject.restart(migrationContext); verify(migratedBlockHashConsumer).accept(Bytes.fromHex("abcd".repeat(24))); - // TODO: fix - // final var expectedInfo = new BlockStreamInfo( - // blockInfo.lastBlockNumber(), - // blockInfo.firstConsTimeOfLastBlock(), - // Bytes.fromHex("dd".repeat(48) + "cc".repeat(48) + "bb".repeat(48) + "aa".repeat(48)), - // Bytes.fromHex("abcd".repeat(24 * 255)), - // Bytes.EMPTY, - // Bytes.EMPTY, - // 0, - // List.of(), - // blockInfo.consTimeOfLastHandledTxn(), - // false, - // SemanticVersion.DEFAULT, - // blockInfo.consTimeOfLastHandledTxn(), - // blockInfo.consTimeOfLastHandledTxn(), - // Bytes.EMPTY, - // Bytes.EMPTY, - // Bytes.EMPTY); - // verify(state).put(expectedInfo); + final var expectedInfo = new BlockStreamInfo( + blockInfo.lastBlockNumber(), + blockInfo.firstConsTimeOfLastBlock(), + Bytes.fromHex("dd".repeat(48) + "cc".repeat(48) + "bb".repeat(48) + "aa".repeat(48)), + Bytes.fromHex("abcd".repeat(24 * 255)), + Bytes.EMPTY, + Bytes.EMPTY, + 0, + List.of(), + blockInfo.consTimeOfLastHandledTxn(), + false, + SemanticVersion.DEFAULT, + blockInfo.consTimeOfLastHandledTxn(), + blockInfo.consTimeOfLastHandledTxn(), + Bytes.EMPTY, + Bytes.EMPTY, + Bytes.EMPTY, + List.of(), + 0); + verify(state).put(expectedInfo); } @Test From 203a3c40bc8f9789c9676cd2d2a9824d09fc4441 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 3 Nov 2025 09:58:31 -0700 Subject: [PATCH 59/63] Test improvement Signed-off-by: Matt Hess --- .../node/app/blocks/impl/BlockStreamManagerImplTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java index 4a456efe237f..cda9716b58dd 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/BlockStreamManagerImplTest.java @@ -1167,7 +1167,8 @@ void blockFooterNotWrittenWhenBlockNotClosed() { final AtomicBoolean footerWritten = new AtomicBoolean(false); - doAnswer(invocationOnMock -> { + lenient() + .doAnswer(invocationOnMock -> { final var item = BlockItem.PROTOBUF.parse((Bytes) invocationOnMock.getArgument(1)); if (item.hasBlockFooter()) { footerWritten.set(true); From e9e4fef1960b43f1d17374a26e234118114c6f9e Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 3 Nov 2025 09:58:55 -0700 Subject: [PATCH 60/63] Replace filtered item hash with filtered single item Signed-off-by: Matt Hess --- .../main/proto/block/stream/block_item.proto | 23 ++++--------------- .../block/BlockContentsValidator.java | 2 +- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto index b41749642be1..83ed42de170f 100644 --- a/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto +++ b/hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_item.proto @@ -133,7 +133,7 @@ import "block/stream/output/block_footer.proto"; * - `trace_data` * - Any subtree (depending on what was filtered). * This item contains it's path in the tree and must be fully parsed. - * - `filtered_item_hash` + * - `filtered_single_item` * - No subtree (and not part of the "proof" merkle tree) * - `block_footer` * - `block_proof` @@ -141,7 +141,7 @@ import "block/stream/output/block_footer.proto"; */ message BlockItem { // Reserved for future items that require separate handling for block hash purposes. - reserved 14,15,16,17,18,19; + reserved 13,14,15,16,17,18,19,20; oneof item { /** @@ -207,19 +207,9 @@ message BlockItem { com.hedera.hapi.block.stream.output.StateChanges state_changes = 7; /** - * Verification data for an item filtered from the stream.
- * This is a hash for a merkle tree node where the contents of that - * part of the merkle tree have been removed from this stream. - *

- * Items of this type SHALL NOT be present in the full (unfiltered) - * block stream.
- * Items of this type SHALL replace any item removed from a partial - * (filtered) block stream.
- * Presence of `filtered_item` entries SHALL NOT prevent verification - * of a block, but MAY preclude verification or reconstruction of - * consensus state.
+ * A block item intentionally filtered from the stream. */ - FilteredItemHash filtered_item_hash = 8 [deprecated = true]; + FilteredSingleItem filtered_single_item = 8; /** * A signed block proof.
@@ -270,11 +260,6 @@ message BlockItem { * One or more Block Proof items SHALL follow this item. */ com.hedera.hapi.block.stream.output.BlockFooter block_footer = 12; - - /** - * A transaction intentionally filtered from the stream. - */ - FilteredSingleItem filtered_single_item = 13; } } diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/BlockContentsValidator.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/BlockContentsValidator.java index 04a15e07a015..57f2347edb8b 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/BlockContentsValidator.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/junit/support/validators/block/BlockContentsValidator.java @@ -105,7 +105,7 @@ private int validateSingleRound(final List items, int startIndex) { final var kind = item.item().kind(); switch (kind) { case EVENT_HEADER, STATE_CHANGES -> hasEventOrStateChange = true; - case RECORD_FILE, FILTERED_ITEM_HASH, FILTERED_SINGLE_ITEM -> + case RECORD_FILE, FILTERED_SINGLE_ITEM -> Assertions.fail("Unexpected item type " + kind + " at index " + currentIndex); default -> { // No-op From 8debbcc8c032d51910f6813b3e01f85c2fdb3a30 Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 3 Nov 2025 11:37:30 -0700 Subject: [PATCH 61/63] Increase timeout Signed-off-by: Matt Hess --- .../blocks/impl/streaming/BlockNodeConnectionManagerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockNodeConnectionManagerTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockNodeConnectionManagerTest.java index 33eed6484def..dad72d06d1db 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockNodeConnectionManagerTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/BlockNodeConnectionManagerTest.java @@ -1365,7 +1365,7 @@ void testStartConfigWatcher_reactsToCreateModifyDelete() throws Exception { // Exercise unchanged path: write back same content and ensure no restart occurs Files.writeString( file, valid, StandardCharsets.UTF_8, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - awaitCondition(() -> !availableNodes().isEmpty(), 2_000); + awaitCondition(() -> !availableNodes().isEmpty(), 5_000); final Map before = new HashMap<>(connections()); invoke_refreshAvailableBlockNodes(); final Map after = new HashMap<>(connections()); From 448c5ff5ebe3ec17ab72777260d4fa77039513da Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 3 Nov 2025 12:01:30 -0700 Subject: [PATCH 62/63] Remove block node suite test (not useful, already covered) Signed-off-by: Matt Hess --- .../bdd/suites/blocknode/BlockNodeSuite.java | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java index 6d02a3d911af..ed3b0c91bc33 100644 --- a/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java +++ b/hedera-node/test-clients/src/main/java/com/hedera/services/bdd/suites/blocknode/BlockNodeSuite.java @@ -9,8 +9,6 @@ import static com.hedera.services.bdd.spec.utilops.UtilVerbs.assertBlockNodeCommsLogContains; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.assertBlockNodeCommsLogContainsTimeframe; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.assertBlockNodeCommsLogDoesNotContain; -import static com.hedera.services.bdd.spec.utilops.UtilVerbs.assertHgcaaLogContainsTimeframe; -import static com.hedera.services.bdd.spec.utilops.UtilVerbs.assertHgcaaLogDoesNotContain; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.doingContextual; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.sourcingContextual; import static com.hedera.services.bdd.spec.utilops.UtilVerbs.waitForActive; @@ -1045,35 +1043,4 @@ private Stream validateHappyPath(final int blocksToWait) { "Sending ad hoc request to block node (type=END_OF_BLOCK)", Duration.ofSeconds(0))); } - - @HapiTest - @HapiBlockNode( - networkSize = 1, - blockNodeConfigs = {@BlockNodeConfig(nodeId = 0, mode = BlockNodeMode.SIMULATOR)}, - subProcessNodeConfigs = { - @SubProcessNodeConfig( - nodeId = 0, - blockNodeIds = {0}, - blockNodePriorities = {0}, - applicationPropertiesOverrides = { - "blockStream.streamMode", "BOTH", - "blockStream.writerMode", "FILE_AND_GRPC" - }) - }) - @Order(13) - final Stream node0SendEndOfBlockHappyPath() { - final AtomicReference timeRef = new AtomicReference<>(); - return hapiTest( - doingContextual(spec -> timeRef.set(Instant.now())), - waitUntilNextBlocks(10).withBackgroundTraffic(true), - // assert no errors - assertHgcaaLogDoesNotContain(byNodeId(0), "ERROR", Duration.ofSeconds(5)), - sourcingContextual(spec -> assertHgcaaLogContainsTimeframe( - byNodeId(0), - timeRef::get, - Duration.ofMinutes(1), - Duration.ofMinutes(1), - // Should send END_OF_BLOCK requests - "Sending request to block node (type=END_OF_BLOCK)"))); - } } From df9525a7df0ed63f22916d28cd6101c57416ed1d Mon Sep 17 00:00:00 2001 From: Matt Hess Date: Mon, 3 Nov 2025 12:08:09 -0700 Subject: [PATCH 63/63] Remove stray comment Signed-off-by: Matt Hess --- .../app/blocks/impl/streaming/GrpcBlockItemWriterTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/GrpcBlockItemWriterTest.java b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/GrpcBlockItemWriterTest.java index e5bed6b4db77..aeaa81340fa5 100644 --- a/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/GrpcBlockItemWriterTest.java +++ b/hedera-node/hedera-app/src/test/java/com/hedera/node/app/blocks/impl/streaming/GrpcBlockItemWriterTest.java @@ -73,10 +73,7 @@ void testWritePbjItem() { // Create BlockProof as easiest way to build object from BlockStreams Bytes bytes = Bytes.wrap(new byte[] {1, 2, 3, 4, 5}); final var proof = BlockItem.newBuilder() - .blockProof(BlockProof.newBuilder() - // TODO: add TssSigned or StateProof (includes sig) - // .blockSignature(bytes) - .siblingHashes(new ArrayList<>())) + .blockProof(BlockProof.newBuilder().siblingHashes(new ArrayList<>())) .build(); grpcBlockItemWriter.writePbjItem(proof);