Make syncing performance more steady. (#6722)

* Initial commit. * Update tests. * Update AllTests. * Post-rebase update AllTests. * Improve log statements which was hidden because of logScope issues. * Eliminate duplicate `direction` fields. Debugging blobs map. * Fix blobs shortmap generation. * Add fixes to map generation and add test helpers. * Fix missing dependency. * Adjust callbacks functionality for untrusted sync. * Attempt to fix defect state issue with new flag `setLastStateRoot`. * Make possible fix less intrusive. * Fix forward declaration missing `=`. * Revert attempt to improve performance. * Revert some all the changes. * Remove spaces. * Remove LFS fixes from the PR. * Update AllTests. * Update copyright year. * Update AllTests. * Post-rebase fixes.
status-im · Feb 4, 2025 · ba934af · ba934af
1 parent 426856e
commit ba934af
Show file tree

Hide file tree

Showing 7 changed files with 2,496 additions and 2,169 deletions.
diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md
@@ -984,33 +984,28 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 7/7 Fail: 0/7 Skip: 0/7
 ## SyncManager test suite
 ```diff
-+ Process all unviable blocks                                                                OK
 + [SyncManager] groupBlobs() test                                                            OK
-+ [SyncQueue#Backward] Async unordered push test                                             OK
-+ [SyncQueue#Backward] Async unordered push with rewind test                                 OK
-+ [SyncQueue#Backward] Good response with missing values towards end                         OK
-+ [SyncQueue#Backward] Handle out-of-band sync progress advancement                          OK
-+ [SyncQueue#Backward] Pass through established limits test                                  OK
-+ [SyncQueue#Backward] Smoke test                                                            OK
-+ [SyncQueue#Backward] Start and finish slots equal                                          OK
-+ [SyncQueue#Backward] Two full requests success/fail                                        OK
++ [SyncQueue# & Backward] Combination of missing parent and good blocks [3 peers] test       OK
++ [SyncQueue# & Backward] Failure request push test                                          OK
++ [SyncQueue# & Backward] Invalid block [3 peers] test                                       OK
++ [SyncQueue# & Backward] Smoke [3 peers] test                                               OK
++ [SyncQueue# & Backward] Smoke [single peer] test                                           OK
++ [SyncQueue# & Backward] Unviable block [3 peers] test                                      OK
++ [SyncQueue# & Forward] Combination of missing parent and good blocks [3 peers] test        OK
++ [SyncQueue# & Forward] Failure request push test                                           OK
++ [SyncQueue# & Forward] Invalid block [3 peers] test                                        OK
++ [SyncQueue# & Forward] Smoke [3 peers] test                                                OK
++ [SyncQueue# & Forward] Smoke [single peer] test                                            OK
++ [SyncQueue# & Forward] Unviable block [3 peers] test                                       OK
++ [SyncQueue#Backward] Missing parent and exponential rewind [3 peers] test                  OK
 + [SyncQueue#Backward] getRewindPoint() test                                                 OK
-+ [SyncQueue#Forward] Async unordered push test                                              OK
-+ [SyncQueue#Forward] Async unordered push with rewind test                                  OK
-+ [SyncQueue#Forward] Good response with missing values towards end                          OK
-+ [SyncQueue#Forward] Handle out-of-band sync progress advancement                           OK
-+ [SyncQueue#Forward] Pass through established limits test                                   OK
-+ [SyncQueue#Forward] Smoke test                                                             OK
-+ [SyncQueue#Forward] Start and finish slots equal                                           OK
-+ [SyncQueue#Forward] Two full requests success/fail                                         OK
++ [SyncQueue#Forward] Missing parent and exponential rewind [3 peers] test                   OK
 + [SyncQueue#Forward] getRewindPoint() test                                                  OK
 + [SyncQueue] checkBlobsResponse() test                                                      OK
 + [SyncQueue] checkResponse() test                                                           OK
-+ [SyncQueue] contains() test                                                                OK
-+ [SyncQueue] getLastNonEmptySlot() test                                                     OK
 + [SyncQueue] hasEndGap() test                                                               OK
 ```
-OK: 25/25 Fail: 0/25 Skip: 0/25
+OK: 20/20 Fail: 0/20 Skip: 0/20
 ## Type helpers
 ```diff
 + BeaconBlock                                                                                OK
@@ -1154,4 +1149,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 783/788 Fail: 0/788 Skip: 5/788
+OK: 778/783 Fail: 0/783 Skip: 5/783
diff --git a/beacon_chain/consensus_object_pools/block_clearance.nim b/beacon_chain/consensus_object_pools/block_clearance.nim
@@ -520,10 +520,6 @@ proc addBackfillBlockData*(
             "database corrupt?", clearanceBlock = shortLog(clearanceBlock)
       return err(VerifierError.MissingParent)
 
-    # dag.clearanceState.setStateRoot(trustedStateRoot)
-    # TODO (cheatfate): This is last part of previous TODO comment, we should
-    # set state's `root` to block's `state_root`.
-
     let proposerVerifyTick = Moment.now()
 
     if not(isNil(onStateUpdated)):

diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim
@@ -385,14 +385,11 @@ proc initFullNode(
     else:
       dag.tail.slot
 
-  proc getUntrustedBackfillSlot(): Slot =
+  func getUntrustedBackfillSlot(): Slot =
     if clist.tail.isSome():
       clist.tail.get().blck.slot
     else:
-      getLocalWallSlot()
-
-  func getUntrustedFrontfillSlot(): Slot =
-    getFirstSlotAtFinalizedEpoch()
+      dag.tail.slot
 
   func getFrontfillSlot(): Slot =
     max(dag.frontfill.get(BlockId()).slot, dag.horizon)
@@ -535,7 +532,7 @@ proc initFullNode(
       dag.cfg.DENEB_FORK_EPOCH, dag.cfg.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS,
       SyncQueueKind.Backward, getLocalHeadSlot,
       getLocalWallSlot, getFirstSlotAtFinalizedEpoch, getUntrustedBackfillSlot,
-      getUntrustedFrontfillSlot, isWithinWeakSubjectivityPeriod,
+      getFrontfillSlot, isWithinWeakSubjectivityPeriod,
       clistPivotSlot, untrustedBlockVerifier, maxHeadAge = 0,
       shutdownEvent = node.shutdownEvent,
       flags = syncManagerFlags)