feat: wrapped block validator token tracing for post october 2023 with saved states

tools-and-tests/tools/README.md

@@ -61,7 +61,9 @@ subcommands
 │   ├── json                  # Convert binary Block Stream to JSON
 │   ├── ls                    # List/inspect block files
 │   ├── validate              # Validate block hash chain and signatures
-│   └── wrap                  # Convert record files to wrapped blocks
+│   ├── validate-wrapped      # Validate wrapped blocks with balance checks
+│   ├── wrap                  # Convert record files to wrapped blocks
+│   └── fetchBalanceCheckpoints # Fetch balance checkpoints from GCP
 │
 ├── records                   # Tools for Record Stream files
 │   └── ls                    # List record file info

tools-and-tests/tools/docs/blocks-commands.md

@@ -4,13 +4,14 @@ The `blocks` command contains utilities for working with Block Stream files (.bl
 
 ### Available Subcommands
 
-|      Command       |                                     Description                                     |
-|--------------------|-------------------------------------------------------------------------------------|
-| `json`             | Converts a binary Block Stream to JSON                                              |
-| `ls`               | Prints info for block files (supports .blk, .blk.gz, .blk.zstd, and zip archives)   |
-| `validate`         | Validates a wrapped Block Stream (hash chain and signatures)                        |
-| `validate-wrapped` | Validates wrapped blocks produced by `wrap` (chain, merkle tree, structure, supply) |
-| `wrap`             | Convert record file blocks in day files to wrapped Block Stream blocks              |
+|          Command          |                                     Description                                     |
+|---------------------------|-------------------------------------------------------------------------------------|
+| `json`                    | Converts a binary Block Stream to JSON                                              |
+| `ls`                      | Prints info for block files (supports .blk, .blk.gz, .blk.zstd, and zip archives)   |
+| `validate`                | Validates a wrapped Block Stream (hash chain and signatures)                        |
+| `validate-wrapped`        | Validates wrapped blocks produced by `wrap` (chain, merkle tree, structure, supply) |
+| `wrap`                    | Convert record file blocks in day files to wrapped Block Stream blocks              |
+| `fetchBalanceCheckpoints` | Fetch balance checkpoint files from GCP and compile into a resource file            |
 
 ---
 
@@ -129,30 +130,63 @@ Validates wrapped block stream files produced by the `wrap` command. Walks all b
 #### Usage
 
 ```
-blocks validate-wrapped [-n=<network>] [<files>...]
+blocks validate-wrapped [-n=<network>] [--[no-]validate-balances] [--balance-checkpoints=<file>]
+                        [--custom-balances-dir=<dir>] [--balance-check-interval-days=<days>] [<files>...]
 ```
 
 #### Options
 
-|          Option          |                                           Description                                            |
-|--------------------------|--------------------------------------------------------------------------------------------------|
-| `-n`, `--network <name>` | Network name for network-specific validation (`mainnet`, `testnet`, `none`). Default: `mainnet`. |
-| `<files>...`             | Block files, directories, or zip archives to process.                                            |
+|                      Option                      |                                           Description                                            |
+|--------------------------------------------------|--------------------------------------------------------------------------------------------------|
+| `-n`, `--network <name>`                         | Network name for network-specific validation (`mainnet`, `testnet`, `none`). Default: `mainnet`. |
+| `--validate-balances` / `--no-validate-balances` | Enable or disable balance checkpoint validation. Default: enabled.                               |
+| `--balance-checkpoints <file>`                   | Path to pre-fetched balance checkpoints file (`balance_checkpoints.zstd`).                       |
+| `--custom-balances-dir <dir>`                    | Directory containing custom balance files (`accountBalances_{blockNumber}.pb.gz`).               |
+| `--balance-check-interval-days <days>`           | Only validate balance checkpoints every N days (default: 30 = monthly).                          |
+| `<files>...`                                     | Block files, directories, or zip archives to process.                                            |
+
+#### Balance Validation
+
+When balance validation is enabled (default), the command validates computed account balances against pre-fetched balance checkpoints. This ensures the 50 billion HBAR supply is correctly tracked through all transactions.
+
+Balance checkpoints can be loaded from:
+- A compiled checkpoint file created by `fetchBalanceCheckpoints` (recommended)
+- `balance_checkpoints_monthly.zstd` - 32 checkpoints, ~14MB (default, faster)
+- `balance_checkpoints_weekly.zstd` - 136 checkpoints, ~20MB (more thorough)
+- A directory of custom balance files extracted from saved states
+
+The `--balance-check-interval-days` option controls how often checkpoints are validated. The default of 30 days (monthly) provides a good balance between validation coverage and performance. Use smaller intervals for more thorough validation or larger intervals for faster runs.
+
+**Important:** The validation interval can only be as granular as the checkpoints that were fetched.
+For example, if checkpoints were fetched with `--interval-days 30` (monthly), you cannot validate
+weekly since weekly checkpoints don't exist in the file. To validate at a smaller interval, you
+must first re-fetch checkpoints using `fetchBalanceCheckpoints` with a matching `--interval-days`
+value.
 
 #### Notes
 
 - When starting from block 0, a `StreamingHasher` is created to validate the historical block hash merkle tree and a balance map is maintained for 50 billion HBAR supply validation. When starting from a later block, both are skipped because the prior state is unavailable.
 - Supports both individual block files (nested directories of `.blk.zstd`) and zip archives produced by the `wrap` command.
 - Progress is printed every 1000 blocks with an ETA.
+- If no balance checkpoints are loaded, balance validation is automatically skipped with a warning.
 
 #### Example
 
 ```bash
-# Validate wrapped blocks in a directory
+# Validate wrapped blocks in a directory (balance validation enabled by default)
 blocks validate-wrapped /path/to/wrappedBlocks
 
-# Validate with explicit network
-blocks validate-wrapped -n mainnet /path/to/wrappedBlocks
+# Validate with explicit balance checkpoint file
+blocks validate-wrapped --balance-checkpoints data/balance_checkpoints.zstd /path/to/wrappedBlocks
+
+# Validate balances weekly instead of monthly
+blocks validate-wrapped --balance-check-interval-days 7 /path/to/wrappedBlocks
+
+# Skip balance validation for faster runs
+blocks validate-wrapped --no-validate-balances /path/to/wrappedBlocks
+
+# Validate with custom balance files from saved states
+blocks validate-wrapped --custom-balances-dir /path/to/balance_files /path/to/wrappedBlocks
 ```
 
 ---
@@ -217,3 +251,74 @@ blocks wrap -i /path/to/compressedDays -o /path/to/wrappedBlocks
 # Output as individual unzipped files
 blocks wrap -u -i /path/to/compressedDays -o /path/to/wrappedBlocks
 ```
+
+---
+
+### The `fetchBalanceCheckpoints` Subcommand
+
+Fetches balance checkpoint files from GCP, optionally verifies signatures, and compiles them into a single zstd-compressed resource file for offline balance validation. This command downloads the `accountBalances` files from the Hedera mainnet GCP bucket and processes them into a compact binary format.
+
+#### Usage
+
+```
+blocks fetchBalanceCheckpoints [-o=<outputFile>] [--start-day=<date>] [--end-day=<date>]
+                               [--interval-days=<days>] [--interval-hours=<hours>]
+                               [--skip-signatures] [--block-times=<file>] [--address-book=<file>]
+```
+
+#### Options
+
+|           Option           |                                                 Description                                                  |
+|----------------------------|--------------------------------------------------------------------------------------------------------------|
+| `-o`, `--output <file>`    | Output zstd-compressed file path (default: `balance_checkpoints.zstd`).                                      |
+| `--start-day <date>`       | Start day in format `YYYY-MM-DD` (default: `2019-09-13`).                                                    |
+| `--end-day <date>`         | End day in format `YYYY-MM-DD` (default: `2023-10-23`).                                                      |
+| `--interval-days <days>`   | Only include one checkpoint every N days (e.g., 7 for weekly, 30 for monthly). Overrides `--interval-hours`. |
+| `--interval-hours <hours>` | Only include checkpoints at this hour interval (default: 24 = one per day).                                  |
+| `--skip-signatures`        | Skip signature verification (not recommended for production use).                                            |
+| `--block-times <file>`     | Path to `block_times.bin` file for timestamp to block mapping (default: `data/block_times.bin`).             |
+| `--address-book <file>`    | Path to address book history JSON file for signature verification (default: `data/addressBookHistory.json`). |
+| `--gcp-project <project>`  | GCP project for requester-pays bucket access (default: from `GCP_PROJECT_ID` env var).                       |
+| `--cache-dir <dir>`        | Directory for caching downloaded files (default: `data/gcp-cache`).                                          |
+| `--min-node <id>`          | Minimum node account ID (default: 3).                                                                        |
+| `--max-node <id>`          | Maximum node account ID (default: 34).                                                                       |
+
+#### Prerequisites
+
+- **GCP authentication** - Run `gcloud auth application-default login` before using this command
+- **block_times.bin** - Required for mapping timestamps to block numbers
+- **addressBookHistory.json** - Required for signature verification (unless `--skip-signatures` is used)
+
+#### Output Format
+
+The output file is a zstd-compressed binary file containing checkpoint records:
+
+- Block number (8 bytes, long)
+- Account count (4 bytes, int)
+- For each account: accountNum (8 bytes, long) + balance (8 bytes, long)
+
+This format avoids protobuf parsing limits and supports files with millions of accounts.
+
+#### Example
+
+```bash
+# Fetch monthly checkpoints (recommended for validation)
+blocks fetchBalanceCheckpoints --interval-days 30 -o balance_checkpoints.zstd
+
+# Fetch weekly checkpoints for more thorough validation
+blocks fetchBalanceCheckpoints --interval-days 7 -o balance_checkpoints_weekly.zstd
+
+# Fetch checkpoints for a specific date range
+blocks fetchBalanceCheckpoints --start-day 2022-01-01 --end-day 2022-12-31 -o balance_2022.zstd
+
+# Skip signature verification (faster but less secure)
+blocks fetchBalanceCheckpoints --skip-signatures -o balance_checkpoints.zstd
+```
+
+#### Notes
+
+- The command handles large balance files (2M+ accounts) that exceed standard protobuf parsing limits by using a custom wire-format parser.
+- Downloaded files are cached locally to avoid re-downloading on subsequent runs.
+- Signature verification ensures checkpoint integrity but requires an address book history file.
+- The compiled output file can be used with `validate-wrapped --balance-checkpoints` for offline validation.
+- The `--interval-days` value determines the granularity of validation possible. For example, monthly checkpoints (`--interval-days 30`) only allow monthly validation, not weekly. Choose the fetch interval based on your validation needs.

tools-and-tests/tools/src/main/java/org/hiero/block/tools/blocks/BlocksCommand.java

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 package org.hiero.block.tools.blocks;
 
+import org.hiero.block.tools.blocks.wrapped.FetchBalanceCheckpointsCommand;
 import org.hiero.block.tools.blocks.wrapped.ValidateWrappedBlocksCommand;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Model.CommandSpec;
@@ -18,6 +19,7 @@
             ValidateBlocksCommand.class,
             ToWrappedBlocksCommand.class,
             ValidateWrappedBlocksCommand.class,
+            FetchBalanceCheckpointsCommand.class,
         },
         mixinStandardHelpOptions = true)
 public class BlocksCommand implements Runnable {

tools-and-tests/tools/src/main/java/org/hiero/block/tools/blocks/ValidateBlocksCommand.java

@@ -22,6 +22,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Stream;
 import java.util.zip.GZIPInputStream;
 import org.hiero.block.tools.blocks.model.BlockHashCalculator;
 import org.hiero.block.tools.days.model.AddressBookRegistry;
@@ -34,8 +35,8 @@
 /**
  * Validates a wrapped block stream by checking:
  * <ul>
- *   <li>Hash chain continuity - each block's previousBlockRootHash matches computed hash of previous block</li>
- *   <li>First block has 48 zero bytes for previous hash (genesis)</li>
+ *   <li>Hash chain continuity - each block's previousBlockRootHash matches the computed hash of the previous block</li>
+ *   <li>The first block has 48 zero bytes for the previous hash (genesis)</li>
  *   <li>Signature validation - at least 1/3 + 1 of address book nodes must sign</li>
  * </ul>
  *
@@ -50,7 +51,7 @@
  * as the only parameter. The command will automatically find the {@code addressBookHistory.json} file
  * in that directory if not explicitly specified.</p>
  */
-@SuppressWarnings({"CallToPrintStackTrace", "FieldCanBeLocal"})
+@SuppressWarnings({"CallToPrintStackTrace", "FieldCanBeLocal", "DuplicatedCode"})
 @Command(
         name = "validate",
         description = "Validates a wrapped block stream (hash chain and signatures)",
@@ -60,7 +61,7 @@ public class ValidateBlocksCommand implements Runnable {
     /** Zero hash for genesis block (48 bytes of zeros). */
     private static final byte[] ZERO_HASH = new byte[48];
 
-    /** Pattern to extract block number from filename. */
+    /** Pattern to extract a block number from the filename. */
     private static final Pattern BLOCK_FILE_PATTERN = Pattern.compile("^(\\d+)\\.blk(\\.gz|\\.zstd)?$");
 
     @SuppressWarnings("unused")
@@ -112,7 +113,7 @@ public void run() {
             }
         }
 
-        // Load address book registry if signature validation is enabled
+        // Load the address book registry if signature validation is enabled
         AddressBookRegistry addressBookRegistry = null;
         if (!skipSignatures) {
             if (addressBookFile != null && Files.exists(addressBookFile)) {
@@ -203,12 +204,11 @@ public void run() {
 
                 // Print verbose output
                 if (verbose) {
-                    String status = (hashValid && signaturesValid)
-                            ? Ansi.AUTO.string("@|green VALID|@")
-                            : Ansi.AUTO.string("@|red INVALID|@");
-                    System.out.println(String.format(
-                            "Block %d: %s (hash: %s)",
-                            blockNum, status, BlockHashCalculator.shortHash(currentBlockHash)));
+                    String status =
+                            signaturesValid ? Ansi.AUTO.string("@|green VALID|@") : Ansi.AUTO.string("@|red INVALID|@");
+                    System.out.printf(
+                            "Block %d: %s (hash: %s)%n",
+                            blockNum, status, BlockHashCalculator.shortHash(currentBlockHash));
                 }
 
                 blocksValidated.incrementAndGet();
@@ -332,9 +332,7 @@ private boolean validateSignatures(
         try {
             // Get the address book for this block
             NodeAddressBook addressBook = addressBookRegistry.getCurrentAddressBook();
-            if (addressBook == null
-                    || addressBook.nodeAddress() == null
-                    || addressBook.nodeAddress().isEmpty()) {
+            if (addressBook == null || addressBook.nodeAddress().isEmpty()) {
                 if (verbose) {
                     PrettyPrint.clearProgress();
                     System.out.println(Ansi.AUTO.string(
@@ -348,7 +346,7 @@ private boolean validateSignatures(
 
             // Get block signatures from proof
             Bytes blockSig = blockProof.signedBlockProofOrThrow().blockSignature();
-            if (blockSig == null || blockSig.length() == 0) {
+            if (blockSig.length() == 0) {
                 PrettyPrint.clearProgress();
                 System.out.println(Ansi.AUTO.string("@|red Block " + blockNum + ":|@ No signatures in block proof"));
                 signatureErrors.incrementAndGet();
@@ -402,10 +400,10 @@ private List<BlockSource> findBlockSources(File[] files) {
 
         for (File file : files) {
             if (file.isDirectory()) {
-                // Recursively find blocks in directory
+                // Recursively find blocks in the directory
                 findBlocksInDirectory(file.toPath(), sources);
             } else if (file.getName().endsWith(".zip")) {
-                // Find blocks in zip file
+                // Find blocks in a zip file
                 findBlocksInZip(file.toPath(), sources);
             } else {
                 // Single block file
@@ -426,8 +424,8 @@ private List<BlockSource> findBlockSources(File[] files) {
      * @param sources list to add sources to
      */
     private void findBlocksInDirectory(Path dir, List<BlockSource> sources) {
-        try {
-            Files.walk(dir).filter(Files::isRegularFile).forEach(path -> {
+        try (Stream<Path> paths = Files.walk(dir)) {
+            paths.filter(Files::isRegularFile).forEach(path -> {
                 String fileName = path.getFileName().toString();
                 if (fileName.endsWith(".zip")) {
                     findBlocksInZip(path, sources);
@@ -452,13 +450,15 @@ private void findBlocksInDirectory(Path dir, List<BlockSource> sources) {
     private void findBlocksInZip(Path zipPath, List<BlockSource> sources) {
         try (FileSystem zipFs = FileSystems.newFileSystem(zipPath)) {
             for (Path root : zipFs.getRootDirectories()) {
-                Files.walk(root).filter(Files::isRegularFile).forEach(path -> {
-                    String fileName = path.getFileName().toString();
-                    long blockNum = extractBlockNumber(fileName);
-                    if (blockNum >= 0) {
-                        sources.add(new BlockSource(blockNum, zipPath, path.toString()));
-                    }
-                });
+                try (Stream<Path> paths = Files.walk(root)) {
+                    paths.filter(Files::isRegularFile).forEach(path -> {
+                        String fileName = path.getFileName().toString();
+                        long blockNum = extractBlockNumber(fileName);
+                        if (blockNum >= 0) {
+                            sources.add(new BlockSource(blockNum, zipPath, path.toString()));
+                        }
+                    });
+                }
             }
         } catch (IOException e) {
             System.err.println("Error reading zip file " + zipPath + ": " + e.getMessage());
@@ -490,13 +490,13 @@ private byte[] readBlockBytes(BlockSource source) throws IOException {
         byte[] compressedBytes;
 
         if (source.isZipEntry()) {
-            // Read from zip file
+            // Read from a zip file
             try (FileSystem zipFs = FileSystems.newFileSystem(source.filePath())) {
                 Path entryPath = zipFs.getPath(source.zipEntryName());
                 compressedBytes = Files.readAllBytes(entryPath);
             }
         } else {
-            // Read from regular file
+            // Read from a regular file
             compressedBytes = Files.readAllBytes(source.filePath());
         }