Verified Proxy: Pre-fetch state using eth_createAccessList and add caching to EVM calls

fluffy/evm/async_evm.nim

@@ -312,10 +312,7 @@ proc call*(
     callResult =
       ?(await evm.callFetchingState(vmState, header, tx, optimisticStateFetch))
 
-  if callResult.error.len() > 0:
-    err("EVM execution failed: " & callResult.error)
-  else:
-    ok(callResult)
+  ok(callResult)
 
 proc createAccessList*(
     evm: AsyncEvm, header: Header, tx: TransactionArgs, optimisticStateFetch = true

nimbus_verified_proxy/rpc/accounts.nim

@@ -75,7 +75,10 @@ proc getStorageFromProof(
     return err(proofResult.errorMsg)
 
 proc getStorageFromProof(
-    stateRoot: Hash32, requestedSlot: UInt256, proof: ProofResponse
+    stateRoot: Hash32,
+    requestedSlot: UInt256,
+    proof: ProofResponse,
+    storageProofIndex = 0,
 ): Result[UInt256, string] =
   let account =
     ?getAccountFromProof(
@@ -88,10 +91,10 @@ proc getStorageFromProof(
     # return 0 value
     return ok(u256(0))
 
-  if len(proof.storageProof) != 1:
+  if proof.storageProof.len() <= storageProofIndex:
     return err("no storage proof for requested slot")
 
-  let storageProof = proof.storageProof[0]
+  let storageProof = proof.storageProof[storageProofIndex]
 
   if len(storageProof.proof) == 0:
     return err("empty mpt proof for account with not empty storage")
@@ -107,6 +110,12 @@ proc getAccount*(
     blockNumber: base.BlockNumber,
     stateRoot: Root,
 ): Future[Result[Account, string]] {.async: (raises: []).} =
+  let
+    cacheKey = (stateRoot, address)
+    cachedAcc = lcProxy.accountsCache.get(cacheKey)
+  if cachedAcc.isSome():
+    return ok(cachedAcc.get())
+
   info "Forwarding eth_getAccount", blockNumber
 
   let
@@ -121,6 +130,9 @@ proc getAccount*(
       proof.storageHash, proof.accountProof,
     )
 
+  if account.isOk():
+    lcProxy.accountsCache.put(cacheKey, account.get())
+
   return account
 
 proc getCode*(
@@ -137,6 +149,12 @@ proc getCode*(
   if account.codeHash == EMPTY_CODE_HASH:
     return ok(newSeq[byte]())
 
+  let
+    cacheKey = (stateRoot, address)
+    cachedCode = lcProxy.codeCache.get(cacheKey)
+  if cachedCode.isSome():
+    return ok(cachedCode.get())
+
   info "Forwarding eth_getCode", blockNumber
 
   let code =
@@ -148,6 +166,7 @@ proc getCode*(
   # verify the byte code. since we verified the account against
   # the state root we just need to verify the code hash
   if account.codeHash == keccak256(code):
+    lcProxy.codeCache.put(cacheKey, code)
     return ok(code)
   else:
     return err("received code doesn't match the account code hash")
@@ -159,6 +178,12 @@ proc getStorageAt*(
     blockNumber: base.BlockNumber,
     stateRoot: Root,
 ): Future[Result[UInt256, string]] {.async: (raises: []).} =
+  let
+    cacheKey = (stateRoot, address, slot)
+    cachedSlotValue = lcProxy.storageCache.get(cacheKey)
+  if cachedSlotValue.isSome():
+    return ok(cachedSlotValue.get())
+
   info "Forwarding eth_getStorageAt", blockNumber
 
   let
@@ -170,4 +195,74 @@ proc getStorageAt*(
 
     slotValue = getStorageFromProof(stateRoot, slot, proof)
 
+  if slotValue.isOk():
+    lcProxy.storageCache.put(cacheKey, slotValue.get())
+
   return slotValue
+
+proc populateCachesForAccountAndSlots(
+    lcProxy: VerifiedRpcProxy,
+    address: Address,
+    slots: seq[UInt256],
+    blockNumber: base.BlockNumber,
+    stateRoot: Root,
+): Future[Result[void, string]] {.async: (raises: []).} =
+  var slotsToFetch: seq[UInt256]
+  for s in slots:
+    let storageCacheKey = (stateRoot, address, s)
+    if lcProxy.storageCache.get(storageCacheKey).isNone():
+      slotsToFetch.add(s)
+
+  let accountCacheKey = (stateRoot, address)
+
+  if lcProxy.accountsCache.get(accountCacheKey).isNone() or slotsToFetch.len() > 0:
+    let
+      proof =
+        try:
+          await lcProxy.rpcClient.eth_getProof(
+            address, slotsToFetch, blockId(blockNumber)
+          )
+        except CatchableError as e:
+          return err(e.msg)
+      account = getAccountFromProof(
+        stateRoot, proof.address, proof.balance, proof.nonce, proof.codeHash,
+        proof.storageHash, proof.accountProof,
+      )
+
+    if account.isOk():
+      lcProxy.accountsCache.put(accountCacheKey, account.get())
+
+    for i, s in slotsToFetch:
+      let slotValue = getStorageFromProof(stateRoot, s, proof, i)
+
+      if slotValue.isOk():
+        let storageCacheKey = (stateRoot, address, s)
+        lcProxy.storageCache.put(storageCacheKey, slotValue.get())
+
+  ok()
+
+proc populateCachesUsingAccessList*(
+    lcProxy: VerifiedRpcProxy,
+    blockNumber: base.BlockNumber,
+    stateRoot: Root,
+    tx: TransactionArgs,
+): Future[Result[void, string]] {.async: (raises: []).} =
+  let accessListRes: AccessListResult =
+    try:
+      await lcProxy.rpcClient.eth_createAccessList(tx, blockId(blockNumber))
+    except CatchableError as e:
+      return err(e.msg)
+
+  var futs = newSeqOfCap[Future[Result[void, string]]](accessListRes.accessList.len())
+  for accessPair in accessListRes.accessList:
+    let slots = accessPair.storageKeys.mapIt(UInt256.fromBytesBE(it.data))
+    futs.add lcProxy.populateCachesForAccountAndSlots(
+      accessPair.address, slots, blockNumber, stateRoot
+    )
+
+  try:
+    await allFutures(futs)
+  except CatchableError as e:
+    return err(e.msg)
+
+  ok()

nimbus_verified_proxy/rpc/rpc_eth_api.nim

@@ -11,6 +11,7 @@ import
   std/[strutils, algorithm],
   results,
   chronicles,
+  stew/byteutils,
   json_rpc/[rpcserver, rpcclient, rpcproxy],
   eth/common/accounts,
   eth/common/addresses,
@@ -199,19 +200,31 @@ proc installEthApiHandlers*(vp: VerifiedRpcProxy) =
     if tx.to.isNone():
       raise newException(ValueError, "to address is required")
 
-    if blockTag.kind == bidAlias:
-      raise newException(ValueError, "tag not yet implemented")
-
     let
       header = (await vp.getHeaderByTag(blockTag)).valueOr:
         raise newException(ValueError, error)
-
       optimisticStateFetch = optimisticStateFetch.valueOr:
         true
 
+    # Start fetching code to get it in the code cache
+    discard vp.getCode(tx.to.get(), header.number, header.stateRoot)
+
+    # As a performance optimisation we concurrently pre-fetch the state needed
+    # for the call by calling eth_createAccessList and then using the returned
+    # access list keys to fetch the required state using eth_getProof.
+    (await vp.populateCachesUsingAccessList(header.number, header.stateRoot, tx)).isOkOr:
+      raise newException(ValueError, error)
+
     let callResult = (await vp.evm.call(header, tx, optimisticStateFetch)).valueOr:
       raise newException(ValueError, error)
 
+    if callResult.error.len() > 0:
+      raise (ref ApplicationError)(
+        code: 3,
+        msg: callResult.error,
+        data: Opt.some(JsonString("\"" & callResult.output.to0xHex() & "\"")),
+      )
+
     return callResult.output
 
   vp.proxy.rpc("eth_createAccessList") do(
@@ -220,16 +233,21 @@ proc installEthApiHandlers*(vp: VerifiedRpcProxy) =
     if tx.to.isNone():
       raise newException(ValueError, "to address is required")
 
-    if blockTag.kind == bidAlias:
-      raise newException(ValueError, "tag not yet implemented")
-
     let
       header = (await vp.getHeaderByTag(blockTag)).valueOr:
         raise newException(ValueError, error)
-
       optimisticStateFetch = optimisticStateFetch.valueOr:
         true
 
+    # Start fetching code to get it in the code cache
+    discard vp.getCode(tx.to.get(), header.number, header.stateRoot)
+
+    # As a performance optimisation we concurrently pre-fetch the state needed
+    # for the call by calling eth_createAccessList and then using the returned
+    # access list keys to fetch the required state using eth_getProof.
+    (await vp.populateCachesUsingAccessList(header.number, header.stateRoot, tx)).isOkOr:
+      raise newException(ValueError, error)
+
     let (accessList, error, gasUsed) = (
       await vp.evm.createAccessList(header, tx, optimisticStateFetch)
     ).valueOr:
@@ -238,27 +256,31 @@ proc installEthApiHandlers*(vp: VerifiedRpcProxy) =
     return
       AccessListResult(accessList: accessList, error: error, gasUsed: gasUsed.Quantity)
 
-  #  vp.proxy.rpc("eth_estimateGas") do(
-  #    tx: TransactionArgs, blockTag: BlockTag, optimisticStateFetch: Opt[bool]
-  #  ) -> Quantity:
-  #    if tx.to.isNone():
-  #      raise newException(ValueError, "to address is required")
-  #
-  #    if blockTag.kind == bidAlias:
-  #      raise newException(ValueError, "tag not yet implemented")
-  #
-  #    let
-  #      header = (await vp.getHeaderByTag(blockTag)).valueOr:
-  #        raise newException(ValueError, error)
-  #
-  #      optimisticStateFetch = optimisticStateFetch.valueOr:
-  #        true
-  #
-  #    let gasEstimate = (await vp.evm.estimateGas(header, tx, optimisticStateFetch)).valueOr:
-  #      raise newException(ValueError, error)
-  #
-  #    return gasEstimate.Quantity
-  #
+  # vp.proxy.rpc("eth_estimateGas") do(
+  #   tx: TransactionArgs, blockTag: BlockTag, optimisticStateFetch: Opt[bool]
+  # ) -> Quantity:
+  #   if tx.to.isNone():
+  #     raise newException(ValueError, "to address is required")
+
+  #   let
+  #     header = (await vp.getHeaderByTag(blockTag)).valueOr:
+  #       raise newException(ValueError, error)
+  #     optimisticStateFetch = optimisticStateFetch.valueOr:
+  #       true
+
+  #   # Start fetching code to get it in the code cache
+  #   discard vp.getCode(tx.to.get(), header.number, header.stateRoot)
+
+  #   # As a performance optimisation we concurrently pre-fetch the state needed
+  #   # for the call by calling eth_createAccessList and then using the returned
+  #   # access list keys to fetch the required state using eth_getProof.
+  #   (await vp.populateCachesUsingAccessList(header.number, header.stateRoot, tx)).isOkOr:
+  #     raise newException(ValueError, error)
+
+  #   let gasEstimate = (await vp.evm.estimateGas(header, tx, optimisticStateFetch)).valueOr:
+  #     raise newException(ValueError, error)
+
+  #   return gasEstimate.Quantity
 
   #  vp.proxy.rpc("eth_blobBaseFee") do() -> Quantity:
   #    let header = vp.headerStore.latest.valueOr:

nimbus_verified_proxy/types.nim

@@ -8,16 +8,37 @@
 import
   json_rpc/[rpcproxy],
   stint,
+  minilru,
   ./header_store,
   ../fluffy/evm/async_evm,
   ../execution_chain/common/common
 
-type VerifiedRpcProxy* = ref object
-  com*: CommonRef
-  evm*: AsyncEvm
-  proxy*: RpcProxy
-  headerStore*: HeaderStore
-  chainId*: UInt256
+export minilru
+
+const
+  ACCOUNTS_CACHE_SIZE = 128
+  CODE_CACHE_SIZE = 64
+  STORAGE_CACHE_SIZE = 256
+
+type
+  AccountsCacheKey* = (Root, Address)
+  AccountsCache* = LruCache[AccountsCacheKey, Account]
+
+  CodeCacheKey* = (Root, Address)
+  CodeCache* = LruCache[CodeCacheKey, seq[byte]]
+
+  StorageCacheKey* = (Root, Address, UInt256)
+  StorageCache* = LruCache[StorageCacheKey, UInt256]
+
+  VerifiedRpcProxy* = ref object
+    com*: CommonRef
+    evm*: AsyncEvm
+    proxy*: RpcProxy
+    headerStore*: HeaderStore
+    accountsCache*: AccountsCache
+    codeCache*: CodeCache
+    storageCache*: StorageCache
+    chainId*: UInt256
 
 proc new*(
     T: type VerifiedRpcProxy,
@@ -26,4 +47,12 @@ proc new*(
     headerStore: HeaderStore,
     chainId: UInt256,
 ): T =
-  VerifiedRpcProxy(com: com, proxy: proxy, headerStore: headerStore, chainId: chainId)
+  VerifiedRpcProxy(
+    com: com,
+    proxy: proxy,
+    headerStore: headerStore,
+    accountsCache: AccountsCache.init(ACCOUNTS_CACHE_SIZE),
+    codeCache: CodeCache.init(CODE_CACHE_SIZE),
+    storageCache: StorageCache.init(STORAGE_CACHE_SIZE),
+    chainId: chainId,
+  )