[Storehouse] Triggering CI build (No need to review)

cmd/execution_builder.go

@@ -48,6 +48,7 @@ import (
 	"github.com/onflow/flow-go/engine/execution/checker"
 	"github.com/onflow/flow-go/engine/execution/computation"
 	"github.com/onflow/flow-go/engine/execution/computation/committer"
+	"github.com/onflow/flow-go/engine/execution/computation/computer"
 	txmetrics "github.com/onflow/flow-go/engine/execution/computation/metrics"
 	"github.com/onflow/flow-go/engine/execution/ingestion"
 	"github.com/onflow/flow-go/engine/execution/ingestion/fetcher"
@@ -64,6 +65,7 @@ import (
 	"github.com/onflow/flow-go/fvm/storage/snapshot"
 	"github.com/onflow/flow-go/fvm/systemcontracts"
 	"github.com/onflow/flow-go/ledger"
+	"github.com/onflow/flow-go/ledger/complete"
 	"github.com/onflow/flow-go/ledger/complete/wal"
 	ledgerfactory "github.com/onflow/flow-go/ledger/factory"
 	modelbootstrap "github.com/onflow/flow-go/model/bootstrap"
@@ -125,12 +127,13 @@ type ExecutionNode struct {
 
 	ingestionUnit *engine.Unit
 
-	collector      *metrics.ExecutionCollector
-	executionState state.ExecutionState
-	followerState  protocol.FollowerState
-	committee      hotstuff.DynamicCommittee
-	ledgerStorage  ledger.Ledger
-	registerStore  *storehouse.RegisterStore
+	collector         *metrics.ExecutionCollector
+	executionState    state.ExecutionState
+	followerState     protocol.FollowerState
+	committee         hotstuff.DynamicCommittee
+	ledgerStorage     ledger.Ledger            // set iff !exeConf.payloadless
+	payloadlessLedger ledger.PayloadlessLedger // set iff exeConf.payloadless
+	registerStore     *storehouse.RegisterStore
 
 	// storage
 	events          storageerr.Events
@@ -626,7 +629,16 @@ func (exeNode *ExecutionNode) LoadProviderEngine(
 			})
 	}
 
-	ledgerViewCommitter := committer.NewLedgerViewCommitter(exeNode.ledgerStorage, node.Tracer)
+	var ledgerViewCommitter computer.ViewCommitter
+	if exeNode.exeConf.payloadless {
+		ledgerViewCommitter = committer.NewPayloadlessLedgerViewCommitter(
+			exeNode.payloadlessLedger,
+			node.Tracer,
+			complete.DefaultPathFinderVersion,
+		)
+	} else {
+		ledgerViewCommitter = committer.NewLedgerViewCommitter(exeNode.ledgerStorage, node.Tracer)
+	}
 	exeNode.exeConf.computationConfig.TokenTrackingEnabled = exeNode.exeConf.tokenTrackingEnabled
 	manager, err := computation.New(
 		node.Logger,
@@ -801,8 +813,21 @@ func (exeNode *ExecutionNode) LoadExecutionState(
 
 	// migrate execution data for last sealed and executed block
 
+	// In full mode, both args are the same *complete.Ledger; in payloadless
+	// mode the first is the payloadless ledger (narrow LedgerStateChecker)
+	// and the snapshot-source slot is nil because storehouse is required.
+	var stateChecker state.LedgerStateChecker
+	var snapshotLedger ledger.Ledger
+	if exeNode.exeConf.payloadless {
+		stateChecker = exeNode.payloadlessLedger
+		snapshotLedger = nil
+	} else {
+		stateChecker = exeNode.ledgerStorage
+		snapshotLedger = exeNode.ledgerStorage
+	}
 	exeNode.executionState = state.NewExecutionState(
-		exeNode.ledgerStorage,
+		stateChecker,
+		snapshotLedger,
 		exeNode.commits,
 		node.Storage.Blocks,
 		node.Storage.Headers,
@@ -916,7 +941,41 @@ func (exeNode *ExecutionNode) LoadExecutionStateLedger(
 	module.ReadyDoneAware,
 	error,
 ) {
-	// Create ledger using factory
+	// Ledger selection is two independent choices passed to the factory:
+	//   - --payloadless picks the payloadless vs. full ledger (this branch).
+	//   - --ledger-service-addr (Config.LedgerServiceAddr), when set, means this
+	//     node connects to a remote ledger service rather than running a local
+	//     ledger; the factory then returns a gRPC client instead of a local one.
+	// Combined: payloadless + remote address -> remote payloadless client;
+	// payloadless + no address -> local payloadless ledger; likewise for full mode.
+	if exeNode.exeConf.payloadless {
+		// Payloadless mode. ValidateFlags enforces --enable-storehouse,
+		// so the storehouse is the value source for reads.
+		pl, err := ledgerfactory.NewPayloadlessLedger(ledgerfactory.Config{
+			LedgerServiceAddr:     exeNode.exeConf.ledgerServiceAddr,
+			LedgerMaxRequestSize:  exeNode.exeConf.ledgerMaxRequestSize,
+			LedgerMaxResponseSize: exeNode.exeConf.ledgerMaxResponseSize,
+			Triedir:               exeNode.exeConf.triedir,
+			MTrieCacheSize:        exeNode.exeConf.mTrieCacheSize,
+			CheckpointDistance:    exeNode.exeConf.checkpointDistance,
+			CheckpointsToKeep:     exeNode.exeConf.checkpointsToKeep,
+			MetricsRegisterer:     node.MetricsRegisterer,
+			WALMetrics:            exeNode.collector,
+			LedgerMetrics:         exeNode.collector,
+			Logger:                node.Logger,
+		}, exeNode.toTriggerCheckpoint)
+		if err != nil {
+			return nil, fmt.Errorf("could not create payloadless ledger: %w", err)
+		}
+		exeNode.payloadlessLedger = pl
+		// exeNode.ledgerStorage stays nil in payloadless mode; the
+		// LedgerStateChecker slot in state.NewExecutionState receives the
+		// payloadless ledger directly, and the snapshotLedger slot stays
+		// nil because the storehouse is the value source.
+		return pl, nil
+	}
+
+	// Full mode (default): WAL-backed ledger via the factory.
 	ledgerStorage, err := ledgerfactory.NewLedger(ledgerfactory.Config{
 		LedgerServiceAddr:     exeNode.exeConf.ledgerServiceAddr,
 		LedgerMaxRequestSize:  exeNode.exeConf.ledgerMaxRequestSize,
@@ -1426,11 +1485,52 @@ func (exeNode *ExecutionNode) LoadBootstrapper(node *NodeConfig) error {
 
 		// when bootstrapping, the bootstrap folder must have a checkpoint file
 		// we need to cover this file to the trie folder to restore the trie to restore the execution state.
+		//
+		// Note: in payloadless mode the V6 root checkpoint placed here is later
+		// converted to root.checkpoint.v7 by ledgerfactory.NewPayloadlessLedger
+		// before the bundle reads it. Bootstrap itself stays mode-agnostic.
 		err = copyBootstrapState(node.BootstrapDir, exeNode.exeConf.triedir)
 		if err != nil {
 			return fmt.Errorf("could not load bootstrap state from checkpoint file: %w", err)
 		}
 
+		// In payloadless (V7) mode the spork only produces a V6 root.checkpoint.
+		// Convert it to a V7 root checkpoint here so the payloadless ledger can
+		// seed its forest from it on first boot; later restarts reuse this file
+		// (or a newer numbered V7 checkpoint written by the compactor). The
+		// HasRootCheckpointV7 guard keeps a re-entry after an interrupted
+		// bootstrap from hitting ConvertCheckpointV6ToV7's "output exists" check.
+		//
+		// Only nodes running a local payloadless ledger need this: a node using a
+		// remote ledger service (ledgerServiceAddr set) never reads its local trie
+		// dir, and the remote ledger service performs its own V7 bootstrap. Skipping
+		// the conversion avoids a needless full-forest load on remote-ledger nodes.
+		//
+		// TODO: ConvertCheckpointV6ToV7 reads the entire V6 forest into memory
+		// before emitting V7, a memory/time spike at first boot for mainnet-scale
+		// root checkpoints. A future optimization is to convert subtrie-by-subtrie
+		// without loading the whole forest.
+		if exeNode.exeConf.payloadless && exeNode.exeConf.ledgerServiceAddr == "" {
+			triedir := exeNode.exeConf.triedir
+			hasV7Root, err := wal.HasRootCheckpointV7(triedir)
+			if err != nil {
+				return fmt.Errorf("could not check for V7 root checkpoint: %w", err)
+			}
+			if !hasV7Root {
+				err = wal.ConvertCheckpointV6ToV7(
+					triedir,
+					modelbootstrap.FilenameWALRootCheckpoint,
+					triedir,
+					modelbootstrap.FilenameWALRootCheckpoint+wal.V7FileSuffix,
+					node.Logger,
+					16,
+				)
+				if err != nil {
+					return fmt.Errorf("could not convert V6 root checkpoint to V7 for payloadless node: %w", err)
+				}
+			}
+		}
+
 		err = bootstrapper.BootstrapExecutionDatabase(node.StorageLockMgr, node.ProtocolDB, node.RootSeal)
 		if err != nil {
 			return fmt.Errorf("could not bootstrap execution database: %w", err)

cmd/execution_config.go

@@ -74,6 +74,7 @@ type ExecutionConfig struct {
 	// file descriptors causing connection failures.
 	onflowOnlyLNs                      bool
 	enableStorehouse                   bool
+	payloadless                        bool
 	enableBackgroundStorehouseIndexing bool
 	backgroundIndexerHeightsPerSecond  uint64
 	enableChecker                      bool
@@ -154,6 +155,9 @@ func (exeConf *ExecutionConfig) SetupFlags(flags *pflag.FlagSet) {
 
 	flags.BoolVar(&exeConf.onflowOnlyLNs, "temp-onflow-only-lns", false, "do not use unless required. forces node to only request collections from onflow collection nodes")
 	flags.BoolVar(&exeConf.enableStorehouse, "enable-storehouse", false, "enable storehouse to store registers on disk, default is false")
+	flags.BoolVar(&exeConf.payloadless, "payloadless", false,
+		"run the execution node with a payloadless ledger that stores only leaf hashes; "+
+			"register values are read from the storehouse during execution. requires --enable-storehouse.")
 	flags.BoolVar(&exeConf.enableBackgroundStorehouseIndexing, "enable-background-storehouse-indexing", false, "enable background indexing of storehouse data while storehouse is disabled to eliminate downtime when enabling it. default: false.")
 	flags.Uint64Var(&exeConf.backgroundIndexerHeightsPerSecond, "background-indexer-heights-per-second", storehouse.DefaultHeightsPerSecond, fmt.Sprintf("rate limit for background indexer in heights per second. 0 means no rate limiting. default: %v", storehouse.DefaultHeightsPerSecond))
 	flags.BoolVar(&exeConf.enableChecker, "enable-checker", true, "enable checker to check the correctness of the execution result, default is true")
@@ -198,5 +202,13 @@ func (exeConf *ExecutionConfig) ValidateFlags() error {
 	if exeConf.enableStorehouse {
 		exeConf.enableBackgroundStorehouseIndexing = false
 	}
+	// Payloadless requires storehouse: the payloadless ledger does not retain
+	// register values; the storehouse is the only available value source for
+	// both proof reconstruction and snapshot reads.
+	if exeConf.payloadless && !exeConf.enableStorehouse {
+		return errors.New("--payloadless requires --enable-storehouse: " +
+			"the payloadless ledger does not store register values; " +
+			"the storehouse must provide them at execution time")
+	}
 	return nil
 }

cmd/ledger/main.go

@@ -18,9 +18,11 @@ import (
 	"go.uber.org/atomic"
 	"google.golang.org/grpc"
 
+	"github.com/onflow/flow-go/ledger"
 	ledgerfactory "github.com/onflow/flow-go/ledger/factory"
 	ledgerpb "github.com/onflow/flow-go/ledger/protobuf"
 	"github.com/onflow/flow-go/ledger/remote"
+	"github.com/onflow/flow-go/module"
 	"github.com/onflow/flow-go/module/irrecoverable"
 	"github.com/onflow/flow-go/module/metrics"
 )
@@ -35,6 +37,7 @@ var (
 	checkpointDist      = flag.Uint("checkpoint-distance", 100, "Checkpoint distance")
 	checkpointsToKeep   = flag.Uint("checkpoints-to-keep", 3, "Number of checkpoints to keep")
 	logLevel            = flag.String("loglevel", "info", "Log level (panic, fatal, error, warn, info, debug)")
+	payloadless         = flag.Bool("payloadless", false, "Run the ledger service in payloadless mode (stores leaf hashes instead of full payloads; requires a V7 checkpoint in --triedir).")
 	maxRequestSize      = flag.Uint("max-request-size", 1<<30, "Maximum request message size in bytes (default: 1 GiB)")
 	maxResponseSize     = flag.Uint("max-response-size", 1<<30, "Maximum response message size in bytes (default: 1 GiB)")
 )
@@ -72,14 +75,18 @@ func main() {
 		Str("admin_addr", *adminAddr).
 		Uint("metrics_port", *metricsPort).
 		Int("mtrie_cache_size", *mtrieCacheSize).
+		Bool("payloadless", *payloadless).
 		Msg("starting ledger service")
 
 	// Create trigger for manual checkpointing (used by admin command)
 	triggerCheckpointOnNextSegmentFinish := atomic.NewBool(false)
 
-	// Create ledger using factory
+	// Create ledger using factory. The same config drives both modes; the
+	// payloadless flag selects which factory constructor (and gRPC service) is
+	// wired up. A ledger gRPC server registers either the full [remote.Service]
+	// or the [remote.PayloadlessService], never both.
 	metricsCollector := metrics.NewLedgerCollector("ledger", "wal")
-	ledgerStorage, err := ledgerfactory.NewLedger(ledgerfactory.Config{
+	factoryConfig := ledgerfactory.Config{
 		Triedir:            *triedir,
 		MTrieCacheSize:     uint32(*mtrieCacheSize),
 		CheckpointDistance: *checkpointDist,
@@ -88,24 +95,58 @@ func main() {
 		WALMetrics:         metricsCollector,
 		LedgerMetrics:      metricsCollector,
 		Logger:             logger,
-	}, triggerCheckpointOnNextSegmentFinish)
-	if err != nil {
-		logger.Fatal().Err(err).Msg("failed to create ledger")
+	}
+
+	// ledgerStorage is the lifecycle handle used for readiness, health check,
+	// and shutdown regardless of mode. registerService binds the mode-specific
+	// gRPC service onto the server once it is created.
+	var ledgerStorage module.ReadyDoneAware
+	var registerService func(grpcServer *grpc.Server)
+
+	if *payloadless {
+		payloadlessLedger, err := ledgerfactory.NewPayloadlessLedger(factoryConfig, triggerCheckpointOnNextSegmentFinish)
+		if err != nil {
+			logger.Fatal().Err(err).Msg("failed to create payloadless ledger")
+		}
+		ledgerStorage = payloadlessLedger
+		registerService = func(grpcServer *grpc.Server) {
+			ledgerpb.RegisterPayloadlessLedgerServiceServer(grpcServer, remote.NewPayloadlessService(payloadlessLedger, logger))
+		}
+	} else {
+		fullLedger, err := ledgerfactory.NewLedger(factoryConfig, triggerCheckpointOnNextSegmentFinish)
+		if err != nil {
+			logger.Fatal().Err(err).Msg("failed to create ledger")
+		}
+		ledgerStorage = fullLedger
+		registerService = func(grpcServer *grpc.Server) {
+			ledgerpb.RegisterLedgerServiceServer(grpcServer, remote.NewService(fullLedger, logger))
+		}
 	}
 
 	// Wait for ledger to be ready (WAL replay)
 	logger.Info().Msg("waiting for ledger initialization...")
 	<-ledgerStorage.Ready()
 	logger.Info().Msg("ledger ready")
 
+	// Both the full and payloadless ledgers expose state inspection for the
+	// post-startup health check, though only the full ledger declares it on its
+	// public interface; assert it here so the check works in either mode.
+	inspector, ok := ledgerStorage.(interface {
+		StateCount() int
+		StateByIndex(index int) (ledger.State, error)
+	})
+	if !ok {
+		logger.Fatal().Msg("ledger does not support state inspection")
+	}
+
 	// Check if any trie is loaded after startup
-	stateCount := ledgerStorage.StateCount()
+	stateCount := inspector.StateCount()
 	if stateCount == 0 {
 		logger.Fatal().Msg("no trie loaded after startup - no states available")
 	}
 
 	// Get the last trie state for logging
-	lastState, err := ledgerStorage.StateByIndex(-1)
+	lastState, err := inspector.StateByIndex(-1)
 	if err != nil {
 		logger.Fatal().Err(err).Msg("failed to get last state for logging")
 	}
@@ -123,9 +164,8 @@ func main() {
 		grpc.MaxSendMsgSize(int(*maxResponseSize)),
 	)
 
-	// Create and register ledger service
-	ledgerService := remote.NewService(ledgerStorage, logger)
-	ledgerpb.RegisterLedgerServiceServer(grpcServer, ledgerService)
+	// Register the mode-specific ledger service
+	registerService(grpcServer)
 
 	// Create listeners based on provided flags
 	type listenerInfo struct {

cmd/util/cmd/checkpoint-collect-stats/cmd.go

@@ -3,6 +3,7 @@ package checkpoint_collect_stats
 import (
 	"cmp"
 	"encoding/hex"
+	"fmt"
 	"math"
 	"slices"
 	"strings"
@@ -315,6 +316,15 @@ func getPayloadStatsFromCheckpoint(payloadCallBack func(payload *ledger.Payload)
 	memAllocBefore := debug.GetHeapAllocsBytes()
 	log.Info().Msgf("loading checkpoint(s) from %v", flagCheckpointDir)
 
+	// checkpoint-collect-stats analyzes payload contents (register types, sizes,
+	// account info). V7 (payloadless) checkpoints store only leaf hashes and contain
+	// no payloads, so they cannot be processed here. The WAL replay below loads only
+	// V6 checkpoints and silently ignores V7 files, which would otherwise produce
+	// misleading (stale or empty) stats. Fail fast with a clear error instead.
+	if err := requireV6Checkpoint(flagCheckpointDir); err != nil {
+		log.Fatal().Err(err).Msg("cannot collect stats from checkpoint")
+	}
+
 	diskWal, err := wal.NewDiskWAL(zerolog.Nop(), nil, &metrics.NoopCollector{}, flagCheckpointDir, complete.DefaultCacheSize, pathfinder.PathByteSize, wal.SegmentSize)
 	if err != nil {
 		log.Fatal().Err(err).Msg("cannot create WAL")
@@ -369,6 +379,33 @@ func getPayloadStatsFromCheckpoint(payloadCallBack func(payload *ledger.Payload)
 	return ledgerStats
 }
 
+// requireV6Checkpoint returns an error if the latest checkpoint in dir is a V7
+// (payloadless) checkpoint. checkpoint-collect-stats requires full payloads,
+// which V7 checkpoints do not contain.
+//
+// Only numbered checkpoints are considered (the WAL bootstrap loads the latest
+// numbered V6 checkpoint). If the latest numbered checkpoint is V7, this command
+// would otherwise silently fall back to an older V6 checkpoint or an empty state,
+// reporting misleading stats.
+//
+// Expected error returns during normal operation:
+//   - an error when the latest checkpoint in dir is a V7 (payloadless) checkpoint
+func requireV6Checkpoint(dir string) error {
+	_, latest, err := wal.ListCheckpointsWithInfo(dir)
+	if err != nil {
+		return fmt.Errorf("cannot list checkpoints in %s: %w", dir, err)
+	}
+
+	if latest != nil && latest.Version == wal.VersionV7 {
+		return fmt.Errorf(
+			"checkpoint %d in %s is a V7 (payloadless) checkpoint, which contains no payloads; "+
+				"checkpoint-collect-stats requires a V6 checkpoint",
+			latest.Number, dir)
+	}
+
+	return nil
+}
+
 func getRegisterStats(valueSizesByType sizesByType) []RegisterStatsByTypes {
 	domainStats := make([]RegisterStatsByTypes, 0, len(common.AllStorageDomains))
 	var allDomainSizes []float64