diff --git a/src/Nethermind/Nethermind.State.Flat.Test/Sync/Snap/FlatSnapTrieFactoryTests.cs b/src/Nethermind/Nethermind.State.Flat.Test/Sync/Snap/FlatSnapTrieFactoryTests.cs index 46000acbfc85..0a14f151e22b 100644 --- a/src/Nethermind/Nethermind.State.Flat.Test/Sync/Snap/FlatSnapTrieFactoryTests.cs +++ b/src/Nethermind/Nethermind.State.Flat.Test/Sync/Snap/FlatSnapTrieFactoryTests.cs @@ -69,6 +69,17 @@ public void CreateTrees_DoNotClearDatabase() persistence.DidNotReceive().Clear(); } + [Test] + public void RangePhase_NeverCarriesIntoTheNextRun() + { + (FlatSnapTrieFactory flatFactory, _) = Build(); + ISnapTrieFactory factory = flatFactory; + + factory.MarkRangePhaseFinished(); + + Assert.That(factory.IsRangePhaseFinished(), Is.False); + } + [TestCase(true)] [TestCase(false)] public void Factory_CreatesTreesWithoutThrowing_ForBothDoubleWriteFlagValues(bool doubleWriteCheck) diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/PatriciaSnapTrieFactoryTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/PatriciaSnapTrieFactoryTests.cs new file mode 100644 index 000000000000..9277471eb6e0 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/PatriciaSnapTrieFactoryTests.cs @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; +using Nethermind.Core.Test; +using Nethermind.Logging; +using Nethermind.Synchronization.SnapSync; +using Nethermind.Trie; +using NUnit.Framework; + +namespace Nethermind.Synchronization.Test.SnapSync; + +[TestFixture] +public class PatriciaSnapTrieFactoryTests +{ + // Pinned: databases written by earlier versions are read back with this key and value. + private static readonly byte[] AccountProgressKey = "AccountProgressKey"u8.ToArray(); + + [Test] + public void Records_finished_range_phase_in_the_state_db() + { + TestMemDb stateDb = new(); + PatriciaSnapTrieFactory factory = new(new NodeStorage(stateDb), stateDb, LimboLogs.Instance); + Assert.That(factory.IsRangePhaseFinished(), Is.False); + + factory.MarkRangePhaseFinished(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(stateDb[AccountProgressKey], Is.EqualTo(Keccak.MaxValue.BytesToArray())); + Assert.That(stateDb.WasFlushed, Is.True); + } + } + + // The flag shares the state DB with the trie nodes it describes, so neither outlives the other. + [Test] + public void Reads_back_a_range_phase_finished_by_an_earlier_run() + { + TestMemDb stateDb = new(); + PatriciaSnapTrieFactory before = new(new NodeStorage(stateDb), stateDb, LimboLogs.Instance); + before.MarkRangePhaseFinished(); + + PatriciaSnapTrieFactory afterRestart = new(new NodeStorage(stateDb), stateDb, LimboLogs.Instance); + + Assert.That(afterRestart.IsRangePhaseFinished(), Is.True); + } +} diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs index f559eda9fce2..96f1589beebc 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs @@ -7,13 +7,12 @@ using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; -using Nethermind.Core.Test; using Nethermind.Core.Test.Builders; -using Nethermind.Db; using Nethermind.Logging; using Nethermind.State.Snap; using Nethermind.Synchronization.FastSync; using Nethermind.Synchronization.SnapSync; +using NSubstitute; using NUnit.Framework; namespace Nethermind.Synchronization.Test.SnapSync; @@ -137,14 +136,10 @@ public void Will_deque_storage_request_if_high() } [Test] - public void Will_mark_progress_and_flush_when_finished() + public void Will_mark_range_phase_finished_when_ranges_drain() { - BlockTree blockTree = Build.A.BlockTree() - .WithStateRoot(Keccak.EmptyTreeHash) - .OfChainLength(2).TestObject; - TestMemDb memDb = new(); - SyncConfig syncConfig = new TestSyncConfig() { SnapSyncAccountRangePartitionCount = 1 }; - using ProgressTracker progressTracker = new(memDb, syncConfig, new StateSyncPivot(blockTree, syncConfig, LimboLogs.Instance), LimboLogs.Instance); + ISnapTrieFactory snapTrieFactory = Substitute.For(); + using ProgressTracker progressTracker = CreateProgressTracker(snapTrieFactory: snapTrieFactory); progressTracker.IsFinished(out SnapSyncBatch? request); Assert.That(request!.AccountRangeRequest, Is.Not.Null); @@ -154,8 +149,35 @@ public void Will_mark_progress_and_flush_when_finished() bool finished = progressTracker.IsFinished(out _); Assert.That(finished, Is.True); - Assert.That(memDb.WasFlushed, Is.True); - Assert.That(memDb[ProgressTracker.ACC_PROGRESS_KEY], Is.EqualTo(Keccak.MaxValue.BytesToArray())); + snapTrieFactory.Received(1).MarkRangePhaseFinished(); + } + + [Test] + public void Will_skip_account_ranges_when_range_phase_already_finished() + { + ISnapTrieFactory snapTrieFactory = Substitute.For(); + snapTrieFactory.IsRangePhaseFinished().Returns(true); + using ProgressTracker progressTracker = CreateProgressTracker(snapTrieFactory: snapTrieFactory); + + progressTracker.LoadProgress(); + + Assert.That(progressTracker.IsFinished(out SnapSyncBatch? request), Is.True); + Assert.That(request, Is.Null); + } + + // Regression: account ranges must be requested again rather than skipped over a store that was just emptied. + [Test] + public void Will_request_account_ranges_when_range_phase_not_finished() + { + ISnapTrieFactory snapTrieFactory = Substitute.For(); + snapTrieFactory.IsRangePhaseFinished().Returns(false); + using ProgressTracker progressTracker = CreateProgressTracker(snapTrieFactory: snapTrieFactory); + + progressTracker.LoadProgress(); + + Assert.That(progressTracker.IsFinished(out SnapSyncBatch? request), Is.False); + Assert.That(request!.AccountRangeRequest, Is.Not.Null); + request.Dispose(); } [TestCase("0x0000000000000000000000000000000000000000000000000000000000000000", "0x2000000000000000000000000000000000000000000000000000000000000000", null, "0x8fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")] @@ -225,10 +247,10 @@ public void Should_not_partition_storage_request_if_last_processed_more_than_thr Assert.That(batch1?.StorageRangeRequest?.LimitHash, Is.EqualTo(limitHash ?? Keccak.MaxValue)); } - private ProgressTracker CreateProgressTracker(int accountRangePartition = 1, bool enableStorageSplits = false) + private ProgressTracker CreateProgressTracker(int accountRangePartition = 1, bool enableStorageSplits = false, ISnapTrieFactory? snapTrieFactory = null) { BlockTree blockTree = Build.A.BlockTree().WithStateRoot(Keccak.EmptyTreeHash).OfChainLength(2).TestObject; SyncConfig syncConfig = new TestSyncConfig() { SnapSyncAccountRangePartitionCount = accountRangePartition, EnableSnapSyncStorageRangeSplit = enableStorageSplits }; - return new(new MemDb(), syncConfig, new StateSyncPivot(blockTree, syncConfig, LimboLogs.Instance), LimboLogs.Instance); + return new(snapTrieFactory ?? Substitute.For(), syncConfig, new StateSyncPivot(blockTree, syncConfig, LimboLogs.Instance), LimboLogs.Instance); } } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapStateServerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapStateServerTests.cs index 5d8dbcde5460..258ee340660a 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapStateServerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapStateServerTests.cs @@ -62,9 +62,10 @@ internal TrieSnapServerContext(ILastNStateRootTracker? lastNStateRootTracker = n Server = new SnapStateServer(_store.AsReadOnly(), LimboLogs.Instance, lastNStateRootTracker); _clientStateDb = new MemDb(); - using ProgressTracker progressTracker = new(_clientStateDb, new TestSyncConfig(), new StateSyncPivot(null!, new TestSyncConfig(), LimboLogs.Instance), LimboLogs.Instance); INodeStorage nodeStorage = new NodeStorage(_clientStateDb); - SnapProvider = new SnapProvider(progressTracker, new MemDb(), new PatriciaSnapTrieFactory(nodeStorage, LimboLogs.Instance), LimboLogs.Instance); + PatriciaSnapTrieFactory snapTrieFactory = new(nodeStorage, _clientStateDb, LimboLogs.Instance); + using ProgressTracker progressTracker = new(snapTrieFactory, new TestSyncConfig(), new StateSyncPivot(null!, new TestSyncConfig(), LimboLogs.Instance), LimboLogs.Instance); + SnapProvider = new SnapProvider(progressTracker, new MemDb(), snapTrieFactory, LimboLogs.Instance); } public IWriteBatch BeginWriteBatch() => new WriteBatch(this); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapSyncRunnerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapSyncRunnerTests.cs index ce77984acd4e..17d3b0f94fab 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapSyncRunnerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapSyncRunnerTests.cs @@ -5,6 +5,9 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Logging; +using Nethermind.Synchronization.FastSync; using Nethermind.Synchronization.SnapSync; using NSubstitute; using NUnit.Framework; @@ -25,6 +28,15 @@ public async Task Run_invokes_lifecycle_in_order(DispatcherOutcome outcome, Type ISnapTrieFactory factory = Substitute.For(); factory.When(f => f.EnsureInitialize()).Do(_ => calls.Add("EnsureInitialize")); factory.When(f => f.FinalizeSync()).Do(_ => calls.Add("FinalizeSync")); + factory.IsRangePhaseFinished().Returns(_ => + { + calls.Add("LoadProgress"); + return false; + }); + + ISyncConfig syncConfig = Substitute.For(); + syncConfig.SnapSyncAccountRangePartitionCount.Returns(1); + using ProgressTracker progressTracker = new(factory, syncConfig, Substitute.For(), LimboLogs.Instance); using CancellationTokenSource cts = new(); if (outcome == DispatcherOutcome.Cancels) cts.Cancel(); @@ -38,7 +50,7 @@ public async Task Run_invokes_lifecycle_in_order(DispatcherOutcome outcome, Type DispatcherOutcome.Cancels => throw new OperationCanceledException(token), _ => Task.CompletedTask, }; - }, factory); + }, factory, progressTracker); Func act = () => runner.Run(cts.Token); if (expectedException is null) @@ -46,6 +58,6 @@ public async Task Run_invokes_lifecycle_in_order(DispatcherOutcome outcome, Type else Assert.That(async () => await act(), Throws.InstanceOf(expectedException)); - Assert.That(calls, Is.EqualTo(new[] { "EnsureInitialize", "dispatcher", "FinalizeSync" })); + Assert.That(calls, Is.EqualTo(new[] { "EnsureInitialize", "LoadProgress", "dispatcher", "FinalizeSync" })); } } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestSnapTrieFactory.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestSnapTrieFactory.cs index ba8b5d76c0ad..8b4293a090c6 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestSnapTrieFactory.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/TestSnapTrieFactory.cs @@ -12,7 +12,12 @@ internal class TestSnapTrieFactory( Func> createStateTree, Func>? createStorageTree = null) : ISnapTrieFactory { + private bool _rangePhaseFinished; + public ISnapTree CreateStateTree() => createStateTree(); public ISnapTree CreateStorageTree(in ValueHash256 accountPath) => createStorageTree is not null ? createStorageTree() : throw new NotSupportedException("No storage tree factory provided"); + + public bool IsRangePhaseFinished() => _rangePhaseFinished; + public void MarkRangePhaseFinished() => _rangePhaseFinished = true; } diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/ISnapTrieFactory.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/ISnapTrieFactory.cs index d631c14e4e4c..19a154072e9a 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/ISnapTrieFactory.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/ISnapTrieFactory.cs @@ -14,4 +14,10 @@ void FinalizeSync() { } ISnapTree CreateStateTree(); ISnapTree CreateStorageTree(in ValueHash256 accountPath); + + // Marked when the range phase drains, read after EnsureInitialize, so a later run over the same data + // skips the phase. Records nothing by default, so a backend that does not keep its store across runs + // re-requests the ranges instead of skipping them over data it discarded. + bool IsRangePhaseFinished() => false; + void MarkRangePhaseFinished() { } } diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/PatriciaSnapTrieFactory.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/PatriciaSnapTrieFactory.cs index dd594b2fa43c..27228d2e71ed 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/PatriciaSnapTrieFactory.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/PatriciaSnapTrieFactory.cs @@ -1,7 +1,10 @@ // SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Autofac.Features.AttributeFilters; +using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Db; using Nethermind.Logging; using Nethermind.State; using Nethermind.State.Snap; @@ -9,10 +12,27 @@ namespace Nethermind.Synchronization.SnapSync; -public class PatriciaSnapTrieFactory(INodeStorage nodeStorage, ILogManager logManager) : ISnapTrieFactory +public class PatriciaSnapTrieFactory( + INodeStorage nodeStorage, + [KeyFilter(DbNames.State)] IDb stateDb, + ILogManager logManager) : ISnapTrieFactory { + private static readonly byte[] RangePhaseKey = "AccountProgressKey"u8.ToArray(); + private readonly RawScopedTrieStore _stateTrieStore = new(nodeStorage, null); + public bool IsRangePhaseFinished() + { + byte[]? recorded = stateDb.Get(RangePhaseKey); + return recorded is { Length: 32 } && new ValueHash256(recorded) == ValueKeccak.MaxValue; + } + + public void MarkRangePhaseFinished() + { + stateDb.PutSpan(RangePhaseKey, ValueKeccak.MaxValue.Bytes, WriteFlags.DisableWAL); + stateDb.Flush(); + } + public ISnapTree CreateStateTree() { SnapUpperBoundAdapter adapter = new(_stateTrieStore); diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs index 99f77ab8985e..c8d2fc231795 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs @@ -7,13 +7,11 @@ using System.Collections.Generic; using System.Linq; using System.Threading; -using Autofac.Features.AttributeFilters; using Nethermind.Blockchain.Synchronization; using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; -using Nethermind.Db; using Nethermind.Int256; using Nethermind.Logging; using Nethermind.State.Snap; @@ -29,7 +27,6 @@ public class ProgressTracker : IDisposable private const int CODES_BATCH_SIZE = 1_000; public const int HIGH_CODES_QUEUE_SIZE = CODES_BATCH_SIZE * 5; private const uint StorageRangeSplitFactor = 2; - internal static readonly byte[] ACC_PROGRESS_KEY = "AccountProgressKey"u8.ToArray(); // This does not need to be a lot as it spawn other requests. In fact 8 is probably too much. It is severely // bottlenecked by _syncCommit lock in SnapProviderHelper, which in turns is limited by the IO. @@ -47,7 +44,7 @@ public class ProgressTracker : IDisposable private int _activeAccRefreshRequests; private readonly ILogger _logger; - private readonly IDb _db; + private readonly ISnapTrieFactory _snapTrieFactory; string? _lastStateRangesReport; private DateTimeOffset _lastLogTime = DateTimeOffset.MinValue; private readonly TimeSpan _maxTimeBetweenLog = TimeSpan.FromSeconds(5); @@ -67,10 +64,10 @@ public class ProgressTracker : IDisposable private readonly FastSync.IStateSyncPivot _pivot; private readonly bool _enableStorageRangeSplit; - public ProgressTracker([KeyFilter(DbNames.State)] IDb db, ISyncConfig syncConfig, FastSync.IStateSyncPivot pivot, ILogManager? logManager) + public ProgressTracker(ISnapTrieFactory snapTrieFactory, ISyncConfig syncConfig, FastSync.IStateSyncPivot pivot, ILogManager? logManager) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _db = db ?? throw new ArgumentNullException(nameof(db)); + _snapTrieFactory = snapTrieFactory ?? throw new ArgumentNullException(nameof(snapTrieFactory)); _pivot = pivot; @@ -82,9 +79,6 @@ public ProgressTracker([KeyFilter(DbNames.State)] IDb db, ISyncConfig syncConfig _enableStorageRangeSplit = syncConfig.EnableSnapSyncStorageRangeSplit; SetupAccountRangePartition(); - - //TODO: maybe better to move to a init method instead of the constructor - GetSyncProgress(); } private void SetupAccountRangePartition() @@ -191,7 +185,7 @@ public bool IsFinished(out SnapSyncBatch? nextBatch) if (rangePhaseFinished) { _logger.Info("Snap - State Ranges (Phase 1) finished."); - FinishRangePhase(); + _snapTrieFactory.MarkRangePhaseFinished(); } LogRequest(NO_REQUEST); @@ -463,35 +457,16 @@ public bool IsSnapGetRangesFinished() => AccountRangeReadyForRequest.IsEmpty && _activeCodeRequests == 0 && _activeAccRefreshRequests == 0; - private void GetSyncProgress() + public void LoadProgress() { - // Note, as before, the progress actually only store MaxValue or 0. So we can't actually resume - // snap sync on restart. - byte[] progress = _db.Get(ACC_PROGRESS_KEY); - if (progress is { Length: 32 }) - { - ValueHash256 path = new(progress); + if (!_snapTrieFactory.IsRangePhaseFinished()) return; - if (path == ValueKeccak.MaxValue) - { - _logger.Info($"Snap - State Ranges (Phase 1) is finished."); - foreach (KeyValuePair partition in AccountRangePartitions) - { - partition.Value.MoreAccountsToRight = false; - } - AccountRangeReadyForRequest.Clear(); - } - else - { - _logger.Info($"Snap - State Ranges (Phase 1) progress loaded from DB:{path}"); - } + _logger.Info($"Snap - State Ranges (Phase 1) is finished."); + foreach (KeyValuePair partition in AccountRangePartitions) + { + partition.Value.MoreAccountsToRight = false; } - } - - private void FinishRangePhase() - { - _db.PutSpan(ACC_PROGRESS_KEY, ValueKeccak.MaxValue.Bytes, WriteFlags.DisableWAL); - _db.Flush(); + AccountRangeReadyForRequest.Clear(); } public void TrackAccountToHeal(ValueHash256 path) diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index 137a799b581f..38ebcca6295c 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -322,7 +322,7 @@ private RefreshVerifyResult VerifyRefreshedAccount(AccountsAndProofs response, H { // Empty-backed isolated factory: a proof node that cannot be resolved from the proof itself fails // verification instead of being completed from (or racing) the live client state DB. - ISnapTrieFactory factory = new PatriciaSnapTrieFactory(new NodeStorage(new MemDb()), logManager); + ISnapTrieFactory factory = new PatriciaSnapTrieFactory(new NodeStorage(new MemDb()), NullDb.Instance, logManager); result = SnapProviderHelper.VerifyAccountRange(factory, stateRoot, path, path.IncrementPath(), accounts, response.Proofs); } catch (Exception) diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncRunner.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncRunner.cs index 30d8e89c62c5..63d6eb7dffa4 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncRunner.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncRunner.cs @@ -8,14 +8,18 @@ namespace Nethermind.Synchronization.SnapSync; -public class SnapSyncRunner(Func runDispatcher, ISnapTrieFactory snapTrieFactory) : ISnapSyncRunner +public class SnapSyncRunner( + Func runDispatcher, + ISnapTrieFactory snapTrieFactory, + ProgressTracker progressTracker) : ISnapSyncRunner { - public SnapSyncRunner(SimpleDispatcher dispatcher, ISnapTrieFactory snapTrieFactory) - : this(dispatcher.Run, snapTrieFactory) { } + public SnapSyncRunner(SimpleDispatcher dispatcher, ISnapTrieFactory snapTrieFactory, ProgressTracker progressTracker) + : this(dispatcher.Run, snapTrieFactory, progressTracker) { } public async Task Run(CancellationToken token) { snapTrieFactory.EnsureInitialize(); + progressTracker.LoadProgress(); try { await runDispatcher(token);