test(benchmark): flat-state base point-read benchmark (RocksDB vs sorted-arena vs LMDB)

Directory.Packages.props

@@ -27,6 +27,7 @@
     <PackageVersion Include="Grpc.Tools" Version="2.82.0" />
     <PackageVersion Include="JetBrains.Profiler.Api" Version="1.4.11" />
     <PackageVersion Include="KubernetesClient" Version="19.0.2" />
+    <PackageVersion Include="LightningDB" Version="0.22.0" />
     <PackageVersion Include="MathNet.Numerics.FSharp" Version="5.0.0" />
     <PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="10.0.10" />
     <PackageVersion Include="Microsoft.AspNetCore.DataProtection.Extensions" Version="10.0.10" />

src/Nethermind/Nethermind.Benchmark/Nethermind.Benchmark.csproj

@@ -6,11 +6,15 @@
 
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" />
+    <!-- LMDB reference backend for the flat-base point-read benchmark. Benchmark-only — never
+         reference this from shipping projects. -->
+    <PackageReference Include="LightningDB" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Nethermind.Consensus.Ethash\Nethermind.Consensus.Ethash.csproj" />
     <ProjectReference Include="..\Nethermind.Core.Test\Nethermind.Core.Test.csproj" />
     <ProjectReference Include="..\Nethermind.Core\Nethermind.Core.csproj" />
+    <ProjectReference Include="..\Nethermind.Db.Rocks\Nethermind.Db.Rocks.csproj" />
     <ProjectReference Include="..\Nethermind.Network.Stats\Nethermind.Network.Stats.csproj" />
     <ProjectReference Include="..\Nethermind.State.Flat\Nethermind.State.Flat.csproj" />
   </ItemGroup>

src/Nethermind/Nethermind.Benchmark/State/FlatBase/IFlatPointReadBackend.cs

@@ -0,0 +1,29 @@
+// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+
+namespace Nethermind.Benchmarks.State.FlatBase;
+
+/// <summary>
+/// A storage backend under test in the flat-base point-read benchmark: uniform point reads of
+/// account (20-byte key) and storage-slot (52-byte key) records.
+/// </summary>
+public interface IFlatPointReadBackend : IDisposable
+{
+    /// <summary>Open a read session. Sessions are cheap and not thread-safe — each reader thread
+    /// creates (and disposes) its own; for LMDB a session wraps a read transaction.</summary>
+    IFlatReadSession BeginSession();
+}
+
+/// <summary>Per-thread read handle of an <see cref="IFlatPointReadBackend"/>.</summary>
+public interface IFlatReadSession : IDisposable
+{
+    /// <summary>Read the account record at <paramref name="key20"/> into <paramref name="valueOut"/>.</summary>
+    /// <returns>The value length in bytes, or 0 on a miss.</returns>
+    int GetAccount(ReadOnlySpan<byte> key20, Span<byte> valueOut);
+
+    /// <summary>Read the storage-slot record at <paramref name="key52"/> into <paramref name="valueOut"/>.</summary>
+    /// <returns>The value length in bytes, or 0 on a miss.</returns>
+    int GetSlot(ReadOnlySpan<byte> key52, Span<byte> valueOut);
+}

src/Nethermind/Nethermind.Benchmark/State/FlatBase/LmdbFlatBackend.cs

@@ -0,0 +1,110 @@
+// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.IO;
+using LightningDB;
+
+namespace Nethermind.Benchmarks.State.FlatBase;
+
+/// <summary>
+/// LMDB reference backend via the LightningDB binding: two named databases ("account", "storage")
+/// in one environment. Referenced ONLY by this benchmark project — never from shipping projects.
+/// </summary>
+/// <remarks>
+/// Writes append in globally ascending key order (shards are written in order, sorted within), so
+/// <see cref="PutOptions.AppendData"/> bulk-loads the B-tree without page splits. Reads open the
+/// environment with <c>MDB_NOTLS</c> (read transactions per benchmark worker, not per OS thread slot)
+/// and <c>MDB_NORDAHEAD</c> (no OS readahead — the workload is uniform random point reads).
+/// </remarks>
+internal sealed class LmdbFlatBackend : IFlatPointReadBackend
+{
+    private readonly LightningEnvironment _env;
+    private readonly LightningDatabase _accountDb;
+    private readonly LightningDatabase _storageDb;
+
+    private LmdbFlatBackend(LightningEnvironment env, LightningDatabase accountDb, LightningDatabase storageDb)
+    {
+        _env = env;
+        _accountDb = accountDb;
+        _storageDb = storageDb;
+    }
+
+    public static LmdbFlatBackend OpenWrite(string dir, long mapSize) =>
+        Open(dir, mapSize, EnvironmentOpenFlags.None, new DatabaseConfiguration { Flags = DatabaseOpenFlags.Create });
+
+    public static LmdbFlatBackend OpenRead(string dir, long mapSize) =>
+        Open(dir, mapSize,
+            EnvironmentOpenFlags.ReadOnly | EnvironmentOpenFlags.NoThreadLocalStorage | EnvironmentOpenFlags.NoReadAhead,
+            new DatabaseConfiguration());
+
+    private static LmdbFlatBackend Open(string dir, long mapSize, EnvironmentOpenFlags flags, DatabaseConfiguration dbConfig)
+    {
+        Directory.CreateDirectory(dir);
+        LightningEnvironment env = new(dir, new EnvironmentConfiguration
+        {
+            MapSize = mapSize,
+            MaxDatabases = 2,
+            MaxReaders = 512,
+        });
+        env.Open(flags);
+
+        using LightningTransaction tx = env.BeginTransaction(
+            flags.HasFlag(EnvironmentOpenFlags.ReadOnly) ? TransactionBeginFlags.ReadOnly : TransactionBeginFlags.None);
+        LightningDatabase accountDb = tx.OpenDatabase("account", dbConfig);
+        LightningDatabase storageDb = tx.OpenDatabase("storage", dbConfig);
+        Check(tx.Commit());
+        return new LmdbFlatBackend(env, accountDb, storageDb);
+    }
+
+    /// <summary>Bulk-load one shard. Keys must be sorted ascending and follow all previously written
+    /// keys of the database (<see cref="PutOptions.AppendData"/>). One transaction per shard keeps the
+    /// dirty-page list bounded on the full-scale dataset.</summary>
+    public void PutShard(bool storage, byte[][] keys, byte[][] values, int count)
+    {
+        using LightningTransaction tx = _env.BeginTransaction();
+        LightningDatabase db = storage ? _storageDb : _accountDb;
+        for (int i = 0; i < count; i++)
+            Check(tx.Put(db, keys[i], values[i], PutOptions.AppendData));
+        Check(tx.Commit());
+    }
+
+    public IFlatReadSession BeginSession() => new Session(this);
+
+    public void Dispose()
+    {
+        _accountDb.Dispose();
+        _storageDb.Dispose();
+        _env.Dispose();
+    }
+
+    private static void Check(MDBResultCode resultCode)
+    {
+        if (resultCode != MDBResultCode.Success)
+            throw new InvalidOperationException($"LMDB operation failed: {resultCode}");
+    }
+
+    private sealed class Session(LmdbFlatBackend backend) : IFlatReadSession
+    {
+        private readonly LmdbFlatBackend _backend = backend;
+        private readonly LightningTransaction _tx = backend._env.BeginTransaction(TransactionBeginFlags.ReadOnly);
+
+        public int GetAccount(ReadOnlySpan<byte> key20, Span<byte> valueOut) =>
+            Get(_backend._accountDb, key20, valueOut);
+
+        public int GetSlot(ReadOnlySpan<byte> key52, Span<byte> valueOut) =>
+            Get(_backend._storageDb, key52, valueOut);
+
+        private int Get(LightningDatabase db, ReadOnlySpan<byte> key, Span<byte> valueOut)
+        {
+            (MDBResultCode resultCode, MDBValue _, MDBValue value) = _tx.Get(db, key);
+            if (resultCode != MDBResultCode.Success) return 0;
+
+            ReadOnlySpan<byte> span = value.AsSpan();
+            span.CopyTo(valueOut);
+            return span.Length;
+        }
+
+        public void Dispose() => _tx.Dispose();
+    }
+}

src/Nethermind/Nethermind.Benchmark/State/FlatBase/README.md

@@ -0,0 +1,90 @@
+# Flat-base point-read benchmark (Phase 0 gate)
+
+Standalone micro-benchmark comparing cold/warm uniform-random point reads across three storage
+backends holding **byte-identical** synthetic state data:
+
+| Backend | What it is |
+| --- | --- |
+| `RocksDb` | Production flat layout: `ColumnsDb<FlatDbColumns>` with the default `DbConfig` `FlatDb`/`FlatAccountDb`/`FlatStorageDb` option strings and dedicated HyperClockCaches (300 MiB Account / 700 MiB Storage — the 30/70 split of the 1 GiB flat budget applied by `FlatRocksDbConfigAdjuster`). |
+| `SortedArena` | Prototype "MDBX-like" flat base: one mmap'd arena file per kind (`ArenaFile`, `MADV_RANDOM` on Linux) holding one `SortedTable` (see `Nethermind.State.Flat/PersistedSnapshots/Sorted/FORMAT.md`) per key-prefix shard (256 shards), plus a tiny in-memory shard directory. A point read is an index-block seek + one 4 KiB data-page seek. |
+| `Lmdb` | LMDB via the `LightningDB` package (benchmark-only dependency), bulk-loaded in key order with `MDB_APPEND`; read with `MDB_NOTLS | MDB_NORDAHEAD`. |
+
+Keys and values replicate the production flat encodings (`BaseFlatPersistence`): accounts are 20-byte
+truncated address hashes mapping to ~70–90-byte slim-RLP account values; slots are 52-byte split keys
+(`[4B addrHash | 32B slotHash | 16B addrHash]`) mapping to 32-byte values. All values derive
+deterministically from the generation counter, and every benchmark setup re-validates 1000 sampled
+keys byte-for-byte on the selected backend — a mismatch fails the run.
+
+## Dataset
+
+Built once and reused across runs (a `dataset.marker` file records the parameters; changing scale or
+format rebuilds). Controlled by environment variables:
+
+- `NETH_FLATBENCH_DIR` — dataset root (default: `<temp>/neth-flatbench`). Put it on the disk you want
+  to measure.
+- `NETH_FLATBENCH_SCALE`:
+  - `smoke` (default): 100k accounts / 500k slots. CI/local friendly (~100 MB per backend); the smoke
+    numbers are **indicative only** — everything fits in cache.
+  - `full`: 300M accounts / 1.2B slots (≥ 100 GB per backend). Build takes hours and needs a large
+    disk plus tens of GB of RAM headroom for the RocksDB bulk load/compaction. Run it on purpose, on
+    the target Linux box only.
+
+```bash
+NETH_FLATBENCH_SCALE=full NETH_FLATBENCH_DIR=/mnt/nvme/flatbench \
+  dotnet run -c release --project src/Nethermind/Nethermind.Benchmark.Runner -- \
+  -f '*FlatBasePointRead*'
+```
+
+The first benchmark process builds the dataset (spill → per-shard sorted bulk-load of all three
+backends → validation); subsequent processes reuse it.
+
+## Running
+
+Parameters: backend × {hit, guaranteed-miss} × reader threads {1, 8, 32}. Each invocation performs
+8192 reads split across the workers; results are reported per read.
+
+- **Warm**: just run the filter above. In-process caches (RocksDB block cache, OS page cache for the
+  mmap/LMDB) are hot after warmup.
+- **Cold** (the decision numbers): cold means *page-cache-cold*. On Linux, before **each** measured
+  pass:
+
+  ```bash
+  sync; echo 3 | sudo tee /proc/sys/vm/drop_caches
+  ```
+
+  and run a short single-shot job so warmup does not re-warm the cache, one parameter combination at
+  a time, e.g.:
+
+  ```bash
+  dotnet run -c release --project src/Nethermind/Nethermind.Benchmark.Runner -- \
+    -f '*FlatBasePointRead*' --warmupCount 0 --iterationCount 1 --invocationCount 1 --unrollFactor 1
+  ```
+
+  Repeat (drop caches → run) ≥ 5 times per combination and aggregate manually.
+
+## Metrics to record
+
+- p50/p99 per-read latency and reads/s per (backend, hit/miss, threads) — from the per-read means of
+  the repeated cold passes (BDN's in-run percentiles are meaningless for single-shot cold runs).
+- IOPS per read: run `iostat -x 1` on the dataset device during the pass; `r/s ÷ reads/s` gives
+  physical reads per lookup (the arena's core claim is ~1 data-page read per hit; RocksDB pays
+  index/filter misses on top).
+- Peak RSS per backend (cache budget accounting: RocksDB holds a 1 GiB block cache; the arena and
+  LMDB rely on the page cache).
+
+## Go/no-go thresholds (from planning)
+
+On cold uniform-random reads at the `full` scale (≥ 100 GB per backend):
+
+- **Go** if the arena is ≥ **1.8× RocksDB** on cold random hits **and** ≥ **0.85× LMDB**;
+- miss cost must be ≤ **1.2× LMDB** (the arena has no bloom filters — misses still binary-search a
+  shard; if this fails, Phase 1 adds a per-shard filter before any production work);
+- otherwise **no-go**: stop the flat-base workstream and record the numbers.
+
+## Implementation notes
+
+- `Sorted/` contains verbatim copies of the internal `SortedTable` machinery from
+  `Nethermind.State.Flat/PersistedSnapshots/Sorted/` (no `InternalsVisibleTo` for benchmarks; copying
+  beats widening production visibility). Keep them in sync with the originals.
+- `LightningDB` is referenced **only** by `Nethermind.Benchmark` — never add it to shipping projects.
+- Zero production code was changed for this benchmark.

src/Nethermind/Nethermind.Benchmark/State/FlatBase/RocksDbFlatBackend.cs

@@ -0,0 +1,130 @@
+// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Threading;
+using Nethermind.Core;
+using Nethermind.Core.Test;
+using Nethermind.Db;
+using Nethermind.Db.Rocks;
+using Nethermind.Db.Rocks.Config;
+using Nethermind.Logging;
+using Nethermind.State.Flat;
+
+namespace Nethermind.Benchmarks.State.FlatBase;
+
+/// <summary>
+/// The production-tuned RocksDB flat store: a <see cref="ColumnsDb{T}"/> over
+/// <see cref="FlatDbColumns"/> opened with the default <see cref="DbConfig"/> Flat/FlatAccount/FlatStorage
+/// option strings, plus dedicated HyperClockCaches for the Account (300 MiB) and Storage (700 MiB)
+/// columns — the same 30/70 split of the 1 GiB flat block-cache budget that
+/// <c>Nethermind.Init.Modules.FlatRocksDbConfigAdjuster</c> applies (that adjuster is internal, so its
+/// two-column cache wiring is mirrored here). Reads go through a DB snapshot, like
+/// <c>RocksDbPersistence.CreateReader</c>.
+/// </summary>
+internal sealed class RocksDbFlatBackend : IFlatPointReadBackend
+{
+    private const double AccountCacheShare = 0.3;
+    private const double StorageCacheShare = 0.7;
+    private const ulong BlockCacheBudget = 1024UL * 1024 * 1024;
+
+    private readonly HyperClockCacheWrapper _accountCache;
+    private readonly HyperClockCacheWrapper _storageCache;
+    private readonly ColumnsDb<FlatDbColumns> _db;
+    private readonly Lock _snapshotLock = new();
+    private IColumnDbSnapshot<FlatDbColumns> _snapshot;
+    private IReadOnlyKeyValueStore _accountColumn;
+    private IReadOnlyKeyValueStore _storageColumn;
+
+    public RocksDbFlatBackend(string basePath)
+    {
+        _accountCache = new HyperClockCacheWrapper((ulong)(BlockCacheBudget * AccountCacheShare));
+        _storageCache = new HyperClockCacheWrapper((ulong)(BlockCacheBudget * StorageCacheShare));
+
+        DbConfig dbConfig = new();
+        RocksDbConfigFactory baseFactory = new(
+            dbConfig, new PruningConfig(), new TestHardwareInfo(), NullLogManager.Instance);
+        CacheInjectingConfigFactory factory = new(baseFactory, _accountCache, _storageCache);
+
+        _db = new ColumnsDb<FlatDbColumns>(
+            basePath, new DbSettings("Flat", "flat"), dbConfig, factory, NullLogManager.Instance,
+            Enum.GetValues<FlatDbColumns>());
+    }
+
+    public void WriteShard(FlatDbColumns column, byte[][] keys, byte[][] values, int count)
+    {
+        using IColumnsWriteBatch<FlatDbColumns> batch = _db.StartWriteBatch();
+        IWriteBatch columnBatch = batch.GetColumnBatch(column);
+        for (int i = 0; i < count; i++)
+            columnBatch.PutSpan(keys[i], values[i], WriteFlags.DisableWAL);
+    }
+
+    /// <summary>Materialize the bulk load: flush memtables to SSTs and run a full compaction so
+    /// benchmark reads see the steady-state LSM shape rather than a pile of L0 files.</summary>
+    public void FinishWrites()
+    {
+        _db.Flush();
+        _db.Compact();
+        _db.Flush();
+    }
+
+    public IFlatReadSession BeginSession()
+    {
+        // Sessions are opened concurrently by the benchmark's reader threads; guard the one-time
+        // snapshot creation (reads through the snapshot columns are thread-safe).
+        lock (_snapshotLock)
+        {
+            if (_snapshot is null)
+            {
+                _snapshot = ((IColumnsDb<FlatDbColumns>)_db).CreateSnapshot();
+                _accountColumn = _snapshot.GetColumn(FlatDbColumns.Account);
+                _storageColumn = _snapshot.GetColumn(FlatDbColumns.Storage);
+            }
+        }
+
+        return new Session(this);
+    }
+
+    public void Dispose()
+    {
+        _snapshot?.Dispose();
+        _db.Dispose();
+        _accountCache.Dispose();
+        _storageCache.Dispose();
+    }
+
+    private sealed class Session(RocksDbFlatBackend backend) : IFlatReadSession
+    {
+        public int GetAccount(ReadOnlySpan<byte> key20, Span<byte> valueOut) =>
+            backend._accountColumn.Get(key20, valueOut);
+
+        public int GetSlot(ReadOnlySpan<byte> key52, Span<byte> valueOut) =>
+            backend._storageColumn.Get(key52, valueOut);
+
+        public void Dispose() { }
+    }
+
+    /// <summary>Mirror of the internal <c>FlatRocksDbConfigAdjuster</c>: hand the Account and Storage
+    /// columns their dedicated HyperClockCaches, leaving every other option to the production
+    /// <see cref="DbConfig"/> strings resolved by the wrapped factory.</summary>
+    private sealed class CacheInjectingConfigFactory(
+        IRocksDbConfigFactory inner,
+        HyperClockCacheWrapper accountCache,
+        HyperClockCacheWrapper storageCache) : IRocksDbConfigFactory
+    {
+        public IRocksDbConfig GetForDatabase(string databaseName, string columnName)
+        {
+            IRocksDbConfig config = inner.GetForDatabase(databaseName, columnName);
+            IntPtr? cacheHandle = columnName switch
+            {
+                nameof(FlatDbColumns.Account) => accountCache.Handle,
+                nameof(FlatDbColumns.Storage) => storageCache.Handle,
+                _ => null,
+            };
+
+            return cacheHandle is null
+                ? config
+                : new AdjustedRocksdbConfig(config, "", config.WriteBufferSize.GetValueOrDefault(), cacheHandle);
+        }
+    }
+}