From 37217b676a8065ec61d96fc947fce2df9355afa2 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Mon, 13 Jan 2025 11:31:44 +0100 Subject: [PATCH 01/32] Storage cache --- src/Neo/Persistence/ClonedCache.cs | 33 ++++--- src/Neo/Persistence/DataCache.cs | 86 +++++++++++++++---- src/Neo/Persistence/IReadOnlyStore.cs | 5 ++ src/Neo/Persistence/MemorySnapshot.cs | 36 ++++---- src/Neo/Persistence/MemoryStore.cs | 4 +- src/Neo/Persistence/SerializedCache.cs | 50 +++++++++++ src/Neo/Persistence/SnapshotCache.cs | 36 +++++--- src/Neo/SmartContract/Native/NeoToken.cs | 24 +++++- .../LevelDBStore/Plugins/Storage/Snapshot.cs | 5 +- .../LevelDBStore/Plugins/Storage/Store.cs | 5 +- .../RocksDBStore/Plugins/Storage/Snapshot.cs | 5 +- .../RocksDBStore/Plugins/Storage/Store.cs | 4 +- .../Cryptography/MPTTrie/UT_Trie.cs | 1 + 13 files changed, 229 insertions(+), 65 deletions(-) create mode 100644 src/Neo/Persistence/SerializedCache.cs diff --git a/src/Neo/Persistence/ClonedCache.cs b/src/Neo/Persistence/ClonedCache.cs index d89491e2b9..91167279db 100644 --- a/src/Neo/Persistence/ClonedCache.cs +++ b/src/Neo/Persistence/ClonedCache.cs @@ -9,55 +9,68 @@ // Redistribution and use in source and binary forms with or without // modifications are permitted. +#nullable enable + using Neo.SmartContract; +using System; using System.Collections.Generic; namespace Neo.Persistence { class ClonedCache : DataCache { - private readonly DataCache innerCache; + private readonly DataCache _innerCache; public ClonedCache(DataCache innerCache) { - this.innerCache = innerCache; + _innerCache = innerCache; + } + + public override T? GetCached() where T : default + { + return _innerCache.GetCached(); + } + + protected override void SetCacheInternal(Type type, object? value) + { + _innerCache.serializedCacheChanges[type] = value; } protected override void AddInternal(StorageKey key, StorageItem value) { - innerCache.Add(key, value.Clone()); + _innerCache.Add(key, value.Clone()); } protected override void DeleteInternal(StorageKey key) { - innerCache.Delete(key); + _innerCache.Delete(key); } protected override bool ContainsInternal(StorageKey key) { - return innerCache.Contains(key); + return _innerCache.Contains(key); } /// protected override StorageItem GetInternal(StorageKey key) { - return innerCache[key].Clone(); + return _innerCache[key].Clone(); } protected override IEnumerable<(StorageKey, StorageItem)> SeekInternal(byte[] keyOrPreifx, SeekDirection direction) { - foreach (var (key, value) in innerCache.Seek(keyOrPreifx, direction)) + foreach (var (key, value) in _innerCache.Seek(keyOrPreifx, direction)) yield return (key, value.Clone()); } - protected override StorageItem TryGetInternal(StorageKey key) + protected override StorageItem? TryGetInternal(StorageKey key) { - return innerCache.TryGet(key)?.Clone(); + return _innerCache.TryGet(key)?.Clone(); } protected override void UpdateInternal(StorageKey key, StorageItem value) { - innerCache.GetAndChange(key).FromReplica(value); + _innerCache.GetAndChange(key)?.FromReplica(value); } } } diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index eedab96152..90cde480f2 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -9,10 +9,13 @@ // Redistribution and use in source and binary forms with or without // modifications are permitted. +#nullable enable + using Neo.Extensions; using Neo.SmartContract; using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Runtime.CompilerServices; @@ -46,6 +49,7 @@ public class Trackable private readonly Dictionary dictionary = new(); private readonly HashSet changeSet = new(); + internal readonly Dictionary serializedCacheChanges = new(); /// /// Reads a specified entry from the cache. If the entry is not in the cache, it will be automatically loaded from the underlying storage. @@ -59,7 +63,7 @@ public StorageItem this[StorageKey key] { lock (dictionary) { - if (dictionary.TryGetValue(key, out Trackable trackable)) + if (dictionary.TryGetValue(key, out var trackable)) { if (trackable.State == TrackState.Deleted || trackable.State == TrackState.NotFound) throw new KeyNotFoundException(); @@ -79,6 +83,13 @@ public StorageItem this[StorageKey key] } } + /// + /// Get cached item + /// + /// Cached type + /// Serialized cache + public abstract T? GetCached(); + /// /// Adds a new entry to the cache. /// @@ -90,7 +101,7 @@ public void Add(StorageKey key, StorageItem value) { lock (dictionary) { - if (dictionary.TryGetValue(key, out Trackable trackable)) + if (dictionary.TryGetValue(key, out var trackable)) { trackable.Item = value; trackable.State = trackable.State switch @@ -113,6 +124,23 @@ public void Add(StorageKey key, StorageItem value) } } + /// + /// Adds a new entry to the cache. + /// + /// The key of the entry. + /// The data of the entry. + /// Serialization changes + /// The entry has already been cached. + /// Note: This method does not read the internal storage to check whether the record already exists. + public void Add(StorageKey key, StorageItem value, T? serializedChanges = default) + { + lock (dictionary) + { + Add(key, value); + serializedCacheChanges[typeof(T)] = serializedChanges; + } + } + /// /// Adds a new entry to the underlying storage. /// @@ -126,7 +154,7 @@ public void Add(StorageKey key, StorageItem value) public virtual void Commit() { LinkedList deletedItem = new(); - foreach (Trackable trackable in GetChangeSet()) + foreach (var trackable in GetChangeSet()) switch (trackable.State) { case TrackState.Added: @@ -142,11 +170,16 @@ public virtual void Commit() deletedItem.AddFirst(trackable.Key); break; } - foreach (StorageKey key in deletedItem) + foreach (var key in deletedItem) { dictionary.Remove(key); } + foreach (var serialized in serializedCacheChanges) + { + SetCacheInternal(serialized.Key, serialized); + } changeSet.Clear(); + serializedCacheChanges.Clear(); } /// @@ -176,7 +209,7 @@ public void Delete(StorageKey key) { lock (dictionary) { - if (dictionary.TryGetValue(key, out Trackable trackable)) + if (dictionary.TryGetValue(key, out var trackable)) { if (trackable.State == TrackState.Added) { @@ -191,7 +224,7 @@ public void Delete(StorageKey key) } else { - StorageItem item = TryGetInternal(key); + var item = TryGetInternal(key); if (item == null) return; dictionary.Add(key, new Trackable { @@ -210,13 +243,20 @@ public void Delete(StorageKey key) /// The key of the entry. protected abstract void DeleteInternal(StorageKey key); + /// + /// Set cache internal + /// + /// Type + /// Value + protected abstract void SetCacheInternal(Type type, object? value); + /// /// Finds the entries starting with the specified prefix. /// /// The prefix of the key. /// The search direction. /// The entries found with the desired prefix. - public IEnumerable<(StorageKey Key, StorageItem Value)> Find(byte[] key_prefix = null, SeekDirection direction = SeekDirection.Forward) + public IEnumerable<(StorageKey Key, StorageItem Value)> Find(byte[]? key_prefix = null, SeekDirection direction = SeekDirection.Forward) { var seek_prefix = key_prefix; if (direction == SeekDirection.Backward) @@ -294,7 +334,7 @@ public bool Contains(StorageKey key) { lock (dictionary) { - if (dictionary.TryGetValue(key, out Trackable trackable)) + if (dictionary.TryGetValue(key, out var trackable)) return trackable.State != TrackState.Deleted && trackable.State != TrackState.NotFound; return ContainsInternal(key); } @@ -321,11 +361,11 @@ public bool Contains(StorageKey key) /// The key of the entry. /// A delegate used to create the entry if it doesn't exist. If the entry already exists, the factory will not be used. /// The cached data. Or if it doesn't exist and the is not provided. - public StorageItem GetAndChange(StorageKey key, Func factory = null) + public StorageItem? GetAndChange(StorageKey key, Func? factory = null) { lock (dictionary) { - if (dictionary.TryGetValue(key, out Trackable trackable)) + if (dictionary.TryGetValue(key, out var trackable)) { if (trackable.State == TrackState.Deleted || trackable.State == TrackState.NotFound) { @@ -371,6 +411,20 @@ public StorageItem GetAndChange(StorageKey key, Func factory = null } } + /// + /// Reads a specified entry from the cache, and mark it as . If the entry is not in the cache, it will be automatically loaded from the underlying storage. + /// + /// The key of the entry. + /// Serialized cache + /// A delegate used to create the entry if it doesn't exist. If the entry already exists, the factory will not be used. + /// The cached data. Or if it doesn't exist and the is not provided. + public StorageItem? GetAndChange(StorageKey key, T serializedCache, Func? factory = null) + { + var ret = GetAndChange(key, factory); + serializedCacheChanges[typeof(T)] = serializedCache; + return ret; + } + /// /// Reads a specified entry from the cache. If the entry is not in the cache, it will be automatically loaded from the underlying storage. If the entry doesn't exist, the factory will be used to create a new one. /// @@ -381,7 +435,7 @@ public StorageItem GetOrAdd(StorageKey key, Func factory) { lock (dictionary) { - if (dictionary.TryGetValue(key, out Trackable trackable)) + if (dictionary.TryGetValue(key, out var trackable)) { if (trackable.State == TrackState.Deleted || trackable.State == TrackState.NotFound) { @@ -426,7 +480,7 @@ public StorageItem GetOrAdd(StorageKey key, Func factory) /// The key to be sought. /// The direction of seek. /// An enumerator containing all the entries after seeking. - public IEnumerable<(StorageKey Key, StorageItem Value)> Seek(byte[] keyOrPrefix = null, SeekDirection direction = SeekDirection.Forward) + public IEnumerable<(StorageKey Key, StorageItem Value)> Seek(byte[]? keyOrPrefix = null, SeekDirection direction = SeekDirection.Forward) { IEnumerable<(byte[], StorageKey, StorageItem)> cached; HashSet cachedKeySet; @@ -490,11 +544,11 @@ public StorageItem GetOrAdd(StorageKey key, Func factory) /// /// The key of the entry. /// The cached data. Or if it is neither in the cache nor in the underlying storage. - public StorageItem TryGet(StorageKey key) + public StorageItem? TryGet(StorageKey key) { lock (dictionary) { - if (dictionary.TryGetValue(key, out Trackable trackable)) + if (dictionary.TryGetValue(key, out var trackable)) { if (trackable.State == TrackState.Deleted || trackable.State == TrackState.NotFound) return null; @@ -514,7 +568,7 @@ public StorageItem TryGet(StorageKey key) /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryGet(StorageKey key, out StorageItem item) + public bool TryGet(StorageKey key, [NotNullWhen(true)] out StorageItem? item) { item = TryGet(key); return item != null; @@ -525,7 +579,7 @@ public bool TryGet(StorageKey key, out StorageItem item) /// /// The key of the entry. /// The data of the entry. Or if it doesn't exist. - protected abstract StorageItem TryGetInternal(StorageKey key); + protected abstract StorageItem? TryGetInternal(StorageKey key); /// /// Updates an entry in the underlying storage. diff --git a/src/Neo/Persistence/IReadOnlyStore.cs b/src/Neo/Persistence/IReadOnlyStore.cs index f7ee901fa9..250fded556 100644 --- a/src/Neo/Persistence/IReadOnlyStore.cs +++ b/src/Neo/Persistence/IReadOnlyStore.cs @@ -19,6 +19,11 @@ namespace Neo.Persistence /// public interface IReadOnlyStore { + /// + /// Serialized cache + /// + SerializedCache SerializedCache { get; } + /// /// Seeks to the entry with the specified key. /// diff --git a/src/Neo/Persistence/MemorySnapshot.cs b/src/Neo/Persistence/MemorySnapshot.cs index ca772d9127..8366bde33f 100644 --- a/src/Neo/Persistence/MemorySnapshot.cs +++ b/src/Neo/Persistence/MemorySnapshot.cs @@ -10,7 +10,6 @@ // modifications are permitted. using Neo.Extensions; -using Neo.IO; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; @@ -20,29 +19,32 @@ namespace Neo.Persistence { internal class MemorySnapshot : ISnapshot { - private readonly ConcurrentDictionary innerData; - private readonly ImmutableDictionary immutableData; - private readonly ConcurrentDictionary writeBatch; + private readonly ConcurrentDictionary _innerData; + private readonly ImmutableDictionary _immutableData; + private readonly ConcurrentDictionary _writeBatch; - public MemorySnapshot(ConcurrentDictionary innerData) + public SerializedCache SerializedCache { get; } + + public MemorySnapshot(ConcurrentDictionary innerData, SerializedCache serializedCache) { - this.innerData = innerData; - immutableData = innerData.ToImmutableDictionary(ByteArrayEqualityComparer.Default); - writeBatch = new ConcurrentDictionary(ByteArrayEqualityComparer.Default); + _innerData = innerData; + SerializedCache = serializedCache; + _immutableData = innerData.ToImmutableDictionary(ByteArrayEqualityComparer.Default); + _writeBatch = new ConcurrentDictionary(ByteArrayEqualityComparer.Default); } public void Commit() { - foreach (var pair in writeBatch) + foreach (var pair in _writeBatch) if (pair.Value is null) - innerData.TryRemove(pair.Key, out _); + _innerData.TryRemove(pair.Key, out _); else - innerData[pair.Key] = pair.Value; + _innerData[pair.Key] = pair.Value; } public void Delete(byte[] key) { - writeBatch[key] = null; + _writeBatch[key] = null; } public void Dispose() @@ -51,14 +53,14 @@ public void Dispose() public void Put(byte[] key, byte[] value) { - writeBatch[key[..]] = value[..]; + _writeBatch[key[..]] = value[..]; } /// public IEnumerable<(byte[] Key, byte[] Value)> Seek(byte[] keyOrPrefix, SeekDirection direction = SeekDirection.Forward) { ByteArrayComparer comparer = direction == SeekDirection.Forward ? ByteArrayComparer.Default : ByteArrayComparer.Reverse; - IEnumerable> records = immutableData; + IEnumerable> records = _immutableData; if (keyOrPrefix?.Length > 0) records = records.Where(p => comparer.Compare(p.Key, keyOrPrefix) >= 0); records = records.OrderBy(p => p.Key, comparer); @@ -67,18 +69,18 @@ public void Put(byte[] key, byte[] value) public byte[] TryGet(byte[] key) { - immutableData.TryGetValue(key, out byte[] value); + _immutableData.TryGetValue(key, out byte[] value); return value?[..]; } public bool TryGet(byte[] key, out byte[] value) { - return immutableData.TryGetValue(key, out value); + return _immutableData.TryGetValue(key, out value); } public bool Contains(byte[] key) { - return immutableData.ContainsKey(key); + return _immutableData.ContainsKey(key); } } } diff --git a/src/Neo/Persistence/MemoryStore.cs b/src/Neo/Persistence/MemoryStore.cs index cb57210c7d..d22b86c822 100644 --- a/src/Neo/Persistence/MemoryStore.cs +++ b/src/Neo/Persistence/MemoryStore.cs @@ -10,7 +10,6 @@ // modifications are permitted. using Neo.Extensions; -using Neo.IO; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; @@ -24,6 +23,7 @@ namespace Neo.Persistence public class MemoryStore : IStore { private readonly ConcurrentDictionary _innerData = new(ByteArrayEqualityComparer.Default); + public SerializedCache SerializedCache { get; } = new(); [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Delete(byte[] key) @@ -38,7 +38,7 @@ public void Dispose() [MethodImpl(MethodImplOptions.AggressiveInlining)] public ISnapshot GetSnapshot() { - return new MemorySnapshot(_innerData); + return new MemorySnapshot(_innerData, SerializedCache); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/Neo/Persistence/SerializedCache.cs b/src/Neo/Persistence/SerializedCache.cs new file mode 100644 index 0000000000..ebc9aefb4a --- /dev/null +++ b/src/Neo/Persistence/SerializedCache.cs @@ -0,0 +1,50 @@ +// Copyright (C) 2015-2024 The Neo Project. +// +// SerializedCache.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; + +namespace Neo.Persistence +{ + public class SerializedCache + { + private readonly ConcurrentDictionary _cache = new(); + + /// + /// Get cached entry + /// + /// Type + /// Cache + public T? GetCached() + { + if (_cache.TryGetValue(typeof(T), out var ret)) + { + return (T)ret; + } + + return default; + } + + /// + /// Set entry + /// + /// Type + /// Value + public void Set(Type type, object? value) + { + if (value == null) _cache.Remove(type, out _); + else _cache[type] = value; + } + } +} diff --git a/src/Neo/Persistence/SnapshotCache.cs b/src/Neo/Persistence/SnapshotCache.cs index 777e9b5802..314cb89074 100644 --- a/src/Neo/Persistence/SnapshotCache.cs +++ b/src/Neo/Persistence/SnapshotCache.cs @@ -22,8 +22,8 @@ namespace Neo.Persistence /// public class SnapshotCache : DataCache, IDisposable { - private readonly IReadOnlyStore store; - private readonly ISnapshot snapshot; + private readonly IReadOnlyStore _store; + private readonly ISnapshot _snapshot; /// /// Initializes a new instance of the class. @@ -31,58 +31,68 @@ public class SnapshotCache : DataCache, IDisposable /// An to create a readonly cache; or an to create a snapshot cache. public SnapshotCache(IReadOnlyStore store) { - this.store = store; - snapshot = store as ISnapshot; + _store = store; + _snapshot = store as ISnapshot; + } + + public override T? GetCached() where T : default + { + return _store.SerializedCache.GetCached(); + } + + protected override void SetCacheInternal(Type type, object? value) + { + _store.SerializedCache.Set(type, value); } protected override void AddInternal(StorageKey key, StorageItem value) { - snapshot?.Put(key.ToArray(), value.ToArray()); + _snapshot?.Put(key.ToArray(), value.ToArray()); } protected override void DeleteInternal(StorageKey key) { - snapshot?.Delete(key.ToArray()); + _snapshot?.Delete(key.ToArray()); } public override void Commit() { base.Commit(); - snapshot?.Commit(); + _snapshot?.Commit(); } protected override bool ContainsInternal(StorageKey key) { - return store.Contains(key.ToArray()); + return _store.Contains(key.ToArray()); } public void Dispose() { - snapshot?.Dispose(); + _snapshot?.Dispose(); } /// protected override StorageItem GetInternal(StorageKey key) { - if (store.TryGet(key.ToArray(), out var value)) + if (_store.TryGet(key.ToArray(), out var value)) return new(value); throw new KeyNotFoundException(); } protected override IEnumerable<(StorageKey, StorageItem)> SeekInternal(byte[] keyOrPrefix, SeekDirection direction) { - return store.Seek(keyOrPrefix, direction).Select(p => (new StorageKey(p.Key), new StorageItem(p.Value))); + return _store.Seek(keyOrPrefix, direction).Select(p => (new StorageKey(p.Key), new StorageItem(p.Value))); } /// protected override StorageItem TryGetInternal(StorageKey key) { - return store.TryGet(key.ToArray(), out var value) ? new(value) : null; + return _store.TryGet(key.ToArray(), out var value) ? new(value) : null; } protected override void UpdateInternal(StorageKey key, StorageItem value) { - snapshot?.Put(key.ToArray(), value.ToArray()); + _snapshot?.Put(key.ToArray(), value.ToArray()); } } } diff --git a/src/Neo/SmartContract/Native/NeoToken.cs b/src/Neo/SmartContract/Native/NeoToken.cs index f0321897f2..bfc315af27 100644 --- a/src/Neo/SmartContract/Native/NeoToken.cs +++ b/src/Neo/SmartContract/Native/NeoToken.cs @@ -58,6 +58,12 @@ public sealed class NeoToken : FungibleToken private readonly StorageKey _votersCount; private readonly StorageKey _registerPrice; + private class LastGasPerBlock + { + public BigInteger GasPerBlock = 0; + public long Index = 0; + } + [ContractEvent(1, name: "CandidateStateChanged", "pubkey", ContractParameterType.PublicKey, "registered", ContractParameterType.Boolean, @@ -192,7 +198,11 @@ internal override ContractTask InitializeAsync(ApplicationEngine engine, Hardfor var cachedCommittee = new CachedCommittee(engine.ProtocolSettings.StandbyCommittee.Select(p => (p, BigInteger.Zero))); engine.SnapshotCache.Add(CreateStorageKey(Prefix_Committee), new StorageItem(cachedCommittee)); engine.SnapshotCache.Add(_votersCount, new StorageItem(System.Array.Empty())); - engine.SnapshotCache.Add(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(0u), new StorageItem(5 * GAS.Factor)); + engine.SnapshotCache.Add(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(0u), new StorageItem(5 * GAS.Factor), new LastGasPerBlock() + { + GasPerBlock = 5 * GAS.Factor, + Index = 0 + }); engine.SnapshotCache.Add(_registerPrice, new StorageItem(1000 * GAS.Factor)); return Mint(engine, Contract.GetBFTAddress(engine.ProtocolSettings.StandbyValidators), TotalAmount, false); } @@ -272,7 +282,8 @@ private void SetGasPerBlock(ApplicationEngine engine, BigInteger gasPerBlock) if (!CheckCommittee(engine)) throw new InvalidOperationException(); uint index = engine.PersistingBlock.Index + 1; - StorageItem entry = engine.SnapshotCache.GetAndChange(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(index), () => new StorageItem(gasPerBlock)); + var entry = engine.SnapshotCache.GetAndChange(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(index), + new LastGasPerBlock() { GasPerBlock = gasPerBlock, Index = index }, () => new StorageItem(gasPerBlock)); entry.Set(gasPerBlock); } @@ -284,7 +295,14 @@ private void SetGasPerBlock(ApplicationEngine engine, BigInteger gasPerBlock) [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.ReadStates)] public BigInteger GetGasPerBlock(DataCache snapshot) { - return GetSortedGasRecords(snapshot, Ledger.CurrentIndex(snapshot) + 1).First().GasPerBlock; + var end = Ledger.CurrentIndex(snapshot) + 1; + var cached = snapshot.GetCached(); + if (cached != null && cached.Index < end) + { + return cached.GasPerBlock; + } + + return GetSortedGasRecords(snapshot, end).First().GasPerBlock; } [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.States)] diff --git a/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs b/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs index 9b8364604f..dca1cc3f80 100644 --- a/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs +++ b/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs @@ -28,10 +28,13 @@ internal class Snapshot : ISnapshot, IEnumerable> private readonly WriteBatch _batch; private readonly object _lock = new(); - public Snapshot(DB db) + public SerializedCache SerializedCache { get; } + + public Snapshot(DB db, SerializedCache serializedCache) { _db = db; _snapshot = db.CreateSnapshot(); + SerializedCache = serializedCache; _readOptions = new ReadOptions { FillCache = false, Snapshot = _snapshot }; _batch = new WriteBatch(); } diff --git a/src/Plugins/LevelDBStore/Plugins/Storage/Store.cs b/src/Plugins/LevelDBStore/Plugins/Storage/Store.cs index 622b98e858..0ea136084f 100644 --- a/src/Plugins/LevelDBStore/Plugins/Storage/Store.cs +++ b/src/Plugins/LevelDBStore/Plugins/Storage/Store.cs @@ -24,6 +24,8 @@ internal class Store : IStore, IEnumerable> private readonly DB _db; private readonly Options _options; + public SerializedCache SerializedCache { get; } + public Store(string path) { _options = new Options @@ -33,6 +35,7 @@ public Store(string path) CompressionLevel = CompressionType.SnappyCompression, }; _db = DB.Open(path, _options); + SerializedCache = new(); } public void Delete(byte[] key) @@ -47,7 +50,7 @@ public void Dispose() } public ISnapshot GetSnapshot() => - new Snapshot(_db); + new Snapshot(_db, SerializedCache); public void Put(byte[] key, byte[] value) => _db.Put(WriteOptions.Default, key, value); diff --git a/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs b/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs index 64d962afaa..980ec2116e 100644 --- a/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs +++ b/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs @@ -23,10 +23,13 @@ internal class Snapshot : ISnapshot private readonly WriteBatch batch; private readonly ReadOptions options; - public Snapshot(RocksDb db) + public SerializedCache SerializedCache { get; } + + public Snapshot(RocksDb db, SerializedCache serializedCache) { this.db = db; snapshot = db.CreateSnapshot(); + SerializedCache = serializedCache; batch = new WriteBatch(); options = new ReadOptions(); diff --git a/src/Plugins/RocksDBStore/Plugins/Storage/Store.cs b/src/Plugins/RocksDBStore/Plugins/Storage/Store.cs index 68acf002af..4df32e5512 100644 --- a/src/Plugins/RocksDBStore/Plugins/Storage/Store.cs +++ b/src/Plugins/RocksDBStore/Plugins/Storage/Store.cs @@ -21,6 +21,8 @@ internal class Store : IStore { private readonly RocksDb db; + public SerializedCache SerializedCache { get; } = new(); + public Store(string path) { db = RocksDb.Open(Options.Default, Path.GetFullPath(path)); @@ -33,7 +35,7 @@ public void Dispose() public ISnapshot GetSnapshot() { - return new Snapshot(db); + return new Snapshot(db, SerializedCache); } /// diff --git a/tests/Neo.Cryptography.MPTTrie.Tests/Cryptography/MPTTrie/UT_Trie.cs b/tests/Neo.Cryptography.MPTTrie.Tests/Cryptography/MPTTrie/UT_Trie.cs index d7fcb92775..8189bca72a 100644 --- a/tests/Neo.Cryptography.MPTTrie.Tests/Cryptography/MPTTrie/UT_Trie.cs +++ b/tests/Neo.Cryptography.MPTTrie.Tests/Cryptography/MPTTrie/UT_Trie.cs @@ -23,6 +23,7 @@ namespace Neo.Cryptography.MPTTrie.Tests class TestSnapshot : ISnapshot { public Dictionary store = new Dictionary(ByteArrayEqualityComparer.Default); + public SerializedCache SerializedCache { get; } = new(); private byte[] StoreKey(byte[] key) { From f0bfc3be8732d95961823101516d73449363ba57 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Mon, 13 Jan 2025 11:54:49 +0100 Subject: [PATCH 02/32] reduce --- src/Plugins/LevelDBStore/Plugins/Storage/Store.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Plugins/LevelDBStore/Plugins/Storage/Store.cs b/src/Plugins/LevelDBStore/Plugins/Storage/Store.cs index 0ea136084f..abdc1e63ea 100644 --- a/src/Plugins/LevelDBStore/Plugins/Storage/Store.cs +++ b/src/Plugins/LevelDBStore/Plugins/Storage/Store.cs @@ -24,7 +24,7 @@ internal class Store : IStore, IEnumerable> private readonly DB _db; private readonly Options _options; - public SerializedCache SerializedCache { get; } + public SerializedCache SerializedCache { get; } = new(); public Store(string path) { @@ -35,7 +35,6 @@ public Store(string path) CompressionLevel = CompressionType.SnappyCompression, }; _db = DB.Open(path, _options); - SerializedCache = new(); } public void Delete(byte[] key) From 038a7c4159b8e7bcf52a5a540e266725c7c39307 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Mon, 13 Jan 2025 12:04:13 +0100 Subject: [PATCH 03/32] clean --- src/Neo/Persistence/ClonedCache.cs | 7 ++----- src/Neo/Persistence/DataCache.cs | 12 +++++------- src/Neo/Persistence/SerializedCache.cs | 12 +++++++++++- src/Neo/Persistence/SnapshotCache.cs | 7 ++----- src/Neo/SmartContract/Native/NeoToken.cs | 13 ++++++++++--- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/src/Neo/Persistence/ClonedCache.cs b/src/Neo/Persistence/ClonedCache.cs index 91167279db..88757d3c6f 100644 --- a/src/Neo/Persistence/ClonedCache.cs +++ b/src/Neo/Persistence/ClonedCache.cs @@ -21,16 +21,13 @@ class ClonedCache : DataCache { private readonly DataCache _innerCache; + public override SerializedCache SerializedCache => _innerCache.SerializedCache; + public ClonedCache(DataCache innerCache) { _innerCache = innerCache; } - public override T? GetCached() where T : default - { - return _innerCache.GetCached(); - } - protected override void SetCacheInternal(Type type, object? value) { _innerCache.serializedCacheChanges[type] = value; diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index 90cde480f2..b1887f6967 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -51,6 +51,11 @@ public class Trackable private readonly HashSet changeSet = new(); internal readonly Dictionary serializedCacheChanges = new(); + /// + /// Serialized cache + /// + public abstract SerializedCache SerializedCache { get; } + /// /// Reads a specified entry from the cache. If the entry is not in the cache, it will be automatically loaded from the underlying storage. /// @@ -83,13 +88,6 @@ public StorageItem this[StorageKey key] } } - /// - /// Get cached item - /// - /// Cached type - /// Serialized cache - public abstract T? GetCached(); - /// /// Adds a new entry to the cache. /// diff --git a/src/Neo/Persistence/SerializedCache.cs b/src/Neo/Persistence/SerializedCache.cs index ebc9aefb4a..57edf6025b 100644 --- a/src/Neo/Persistence/SerializedCache.cs +++ b/src/Neo/Persistence/SerializedCache.cs @@ -26,7 +26,7 @@ public class SerializedCache /// /// Type /// Cache - public T? GetCached() + public T? Get() { if (_cache.TryGetValue(typeof(T), out var ret)) { @@ -36,6 +36,16 @@ public class SerializedCache return default; } + /// + /// Set entry + /// + /// Type + /// Value + public void Set(T? value) + { + Set(typeof(T), value); + } + /// /// Set entry /// diff --git a/src/Neo/Persistence/SnapshotCache.cs b/src/Neo/Persistence/SnapshotCache.cs index 314cb89074..5f377c3a67 100644 --- a/src/Neo/Persistence/SnapshotCache.cs +++ b/src/Neo/Persistence/SnapshotCache.cs @@ -25,6 +25,8 @@ public class SnapshotCache : DataCache, IDisposable private readonly IReadOnlyStore _store; private readonly ISnapshot _snapshot; + public override SerializedCache SerializedCache => _store.SerializedCache; + /// /// Initializes a new instance of the class. /// @@ -35,11 +37,6 @@ public SnapshotCache(IReadOnlyStore store) _snapshot = store as ISnapshot; } - public override T? GetCached() where T : default - { - return _store.SerializedCache.GetCached(); - } - protected override void SetCacheInternal(Type type, object? value) { _store.SerializedCache.Set(type, value); diff --git a/src/Neo/SmartContract/Native/NeoToken.cs b/src/Neo/SmartContract/Native/NeoToken.cs index bfc315af27..966fb15db3 100644 --- a/src/Neo/SmartContract/Native/NeoToken.cs +++ b/src/Neo/SmartContract/Native/NeoToken.cs @@ -13,7 +13,6 @@ using Neo.Cryptography.ECC; using Neo.Extensions; -using Neo.IO; using Neo.Persistence; using Neo.SmartContract.Iterators; using Neo.VM; @@ -296,13 +295,21 @@ private void SetGasPerBlock(ApplicationEngine engine, BigInteger gasPerBlock) public BigInteger GetGasPerBlock(DataCache snapshot) { var end = Ledger.CurrentIndex(snapshot) + 1; - var cached = snapshot.GetCached(); + var cached = snapshot.SerializedCache.Get(); if (cached != null && cached.Index < end) { return cached.GasPerBlock; } - return GetSortedGasRecords(snapshot, end).First().GasPerBlock; + var last = GetSortedGasRecords(snapshot, end).First(); + + if (cached == null) + { + // Not cached + snapshot.SerializedCache.Set(new LastGasPerBlock() { Index = last.Index, GasPerBlock = last.GasPerBlock }); + } + + return last.GasPerBlock; } [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.States)] From ccce7a446d15a508cca45a060e340fbf71af0b6c Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Mon, 13 Jan 2025 12:11:11 +0100 Subject: [PATCH 04/32] Fix discard --- src/Neo/Persistence/ClonedCache.cs | 2 +- src/Neo/Persistence/DataCache.cs | 39 +++++++++++++++++++++--- src/Neo/SmartContract/Native/NeoToken.cs | 2 +- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/Neo/Persistence/ClonedCache.cs b/src/Neo/Persistence/ClonedCache.cs index 88757d3c6f..75904df18e 100644 --- a/src/Neo/Persistence/ClonedCache.cs +++ b/src/Neo/Persistence/ClonedCache.cs @@ -30,7 +30,7 @@ public ClonedCache(DataCache innerCache) protected override void SetCacheInternal(Type type, object? value) { - _innerCache.serializedCacheChanges[type] = value; + _innerCache.AddToCache(type, value); } protected override void AddInternal(StorageKey key, StorageItem value) diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index b1887f6967..cc6d88850f 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -49,7 +49,7 @@ public class Trackable private readonly Dictionary dictionary = new(); private readonly HashSet changeSet = new(); - internal readonly Dictionary serializedCacheChanges = new(); + private readonly Dictionary _serializedCacheChanges = new(); /// /// Serialized cache @@ -135,7 +135,36 @@ public void Add(StorageKey key, StorageItem value, T? serializedChanges = def lock (dictionary) { Add(key, value); - serializedCacheChanges[typeof(T)] = serializedChanges; + _serializedCacheChanges[typeof(T)] = serializedChanges; + } + } + + /// + /// Adds a new entry to the cache. + /// + /// Serialization changes + /// The entry has already been cached. + /// Note: This method does not read the internal storage to check whether the record already exists. + public void AddToCache(T? serializedChanges = default) + { + lock (dictionary) + { + _serializedCacheChanges[typeof(T)] = serializedChanges; + } + } + + /// + /// Adds a new entry to the cache. + /// + /// Cache type + /// Serialization changes + /// The entry has already been cached. + /// Note: This method does not read the internal storage to check whether the record already exists. + internal void AddToCache(Type type, object? serializedChanges) + { + lock (dictionary) + { + _serializedCacheChanges[type] = serializedChanges; } } @@ -172,12 +201,12 @@ public virtual void Commit() { dictionary.Remove(key); } - foreach (var serialized in serializedCacheChanges) + foreach (var serialized in _serializedCacheChanges) { SetCacheInternal(serialized.Key, serialized); } changeSet.Clear(); - serializedCacheChanges.Clear(); + _serializedCacheChanges.Clear(); } /// @@ -419,7 +448,7 @@ public bool Contains(StorageKey key) public StorageItem? GetAndChange(StorageKey key, T serializedCache, Func? factory = null) { var ret = GetAndChange(key, factory); - serializedCacheChanges[typeof(T)] = serializedCache; + _serializedCacheChanges[typeof(T)] = serializedCache; return ret; } diff --git a/src/Neo/SmartContract/Native/NeoToken.cs b/src/Neo/SmartContract/Native/NeoToken.cs index 966fb15db3..f06b36fcd8 100644 --- a/src/Neo/SmartContract/Native/NeoToken.cs +++ b/src/Neo/SmartContract/Native/NeoToken.cs @@ -306,7 +306,7 @@ public BigInteger GetGasPerBlock(DataCache snapshot) if (cached == null) { // Not cached - snapshot.SerializedCache.Set(new LastGasPerBlock() { Index = last.Index, GasPerBlock = last.GasPerBlock }); + snapshot.AddToCache(new LastGasPerBlock() { Index = last.Index, GasPerBlock = last.GasPerBlock }); } return last.GasPerBlock; From cca66bad9f581f1ca5c43157ca8196879460f478 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Mon, 20 Jan 2025 10:49:01 +0100 Subject: [PATCH 05/32] Clean --- src/Neo/Persistence/DataCache.cs | 13 ++++++------- src/Neo/Persistence/IStorageCacheEntry.cs | 20 ++++++++++++++++++++ src/Neo/Persistence/SerializedCache.cs | 2 +- src/Neo/SmartContract/Native/NeoToken.cs | 6 ++++-- 4 files changed, 31 insertions(+), 10 deletions(-) create mode 100644 src/Neo/Persistence/IStorageCacheEntry.cs diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index 705bca13a1..ba8d59dc6b 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -117,15 +117,14 @@ public void Add(StorageKey key, StorageItem value) /// /// The key of the entry. /// The data of the entry. - /// Serialization changes /// The entry has already been cached. /// Note: This method does not read the internal storage to check whether the record already exists. - public void Add(StorageKey key, StorageItem value, T? serializedChanges = default) + public void Add(StorageKey key, T value) where T : IStorageCacheEntry { - lock (dictionary) + lock (_dictionary) { - Add(key, value); - _serializedCacheChanges[typeof(T)] = serializedChanges; + Add(key, value.GetStorageItem()); + _serializedCacheChanges[typeof(T)] = value; } } @@ -137,7 +136,7 @@ public void Add(StorageKey key, StorageItem value, T? serializedChanges = def /// Note: This method does not read the internal storage to check whether the record already exists. public void AddToCache(T? serializedChanges = default) { - lock (dictionary) + lock (_dictionary) { _serializedCacheChanges[typeof(T)] = serializedChanges; } @@ -152,7 +151,7 @@ public void AddToCache(T? serializedChanges = default) /// Note: This method does not read the internal storage to check whether the record already exists. internal void AddToCache(Type type, object? serializedChanges) { - lock (dictionary) + lock (_dictionary) { _serializedCacheChanges[type] = serializedChanges; } diff --git a/src/Neo/Persistence/IStorageCacheEntry.cs b/src/Neo/Persistence/IStorageCacheEntry.cs new file mode 100644 index 0000000000..ddddef0711 --- /dev/null +++ b/src/Neo/Persistence/IStorageCacheEntry.cs @@ -0,0 +1,20 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// IStorageCacheEntry.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.SmartContract; + +namespace Neo.Persistence +{ + public interface IStorageCacheEntry + { + public StorageItem GetStorageItem(); + } +} diff --git a/src/Neo/Persistence/SerializedCache.cs b/src/Neo/Persistence/SerializedCache.cs index 57edf6025b..08710bc27c 100644 --- a/src/Neo/Persistence/SerializedCache.cs +++ b/src/Neo/Persistence/SerializedCache.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2015-2024 The Neo Project. +// Copyright (C) 2015-2025 The Neo Project. // // SerializedCache.cs file belongs to the neo project and is free // software distributed under the MIT software license, see the diff --git a/src/Neo/SmartContract/Native/NeoToken.cs b/src/Neo/SmartContract/Native/NeoToken.cs index 5283b93310..c0d4b4c33b 100644 --- a/src/Neo/SmartContract/Native/NeoToken.cs +++ b/src/Neo/SmartContract/Native/NeoToken.cs @@ -57,10 +57,12 @@ public sealed class NeoToken : FungibleToken private readonly StorageKey _votersCount; private readonly StorageKey _registerPrice; - private class LastGasPerBlock + private class LastGasPerBlock : IStorageCacheEntry { public BigInteger GasPerBlock = 0; public long Index = 0; + + public StorageItem GetStorageItem() => new(GasPerBlock); } [ContractEvent(1, name: "CandidateStateChanged", @@ -197,7 +199,7 @@ internal override ContractTask InitializeAsync(ApplicationEngine engine, Hardfor var cachedCommittee = new CachedCommittee(engine.ProtocolSettings.StandbyCommittee.Select(p => (p, BigInteger.Zero))); engine.SnapshotCache.Add(CreateStorageKey(Prefix_Committee), new StorageItem(cachedCommittee)); engine.SnapshotCache.Add(_votersCount, new StorageItem(System.Array.Empty())); - engine.SnapshotCache.Add(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(0u), new StorageItem(5 * GAS.Factor), new LastGasPerBlock() + engine.SnapshotCache.Add(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(0u), new LastGasPerBlock() { GasPerBlock = 5 * GAS.Factor, Index = 0 From 54901edd7623de27401d07e1280a27734fd8b595 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Mon, 20 Jan 2025 11:01:36 +0100 Subject: [PATCH 06/32] clean --- src/Neo/Persistence/DataCache.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index ba8d59dc6b..97f6731c02 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -131,14 +131,14 @@ public void Add(StorageKey key, T value) where T : IStorageCacheEntry /// /// Adds a new entry to the cache. /// - /// Serialization changes + /// The data of the entry. /// The entry has already been cached. /// Note: This method does not read the internal storage to check whether the record already exists. - public void AddToCache(T? serializedChanges = default) + public void AddToCache(T? value = default) where T : IStorageCacheEntry { lock (_dictionary) { - _serializedCacheChanges[typeof(T)] = serializedChanges; + _serializedCacheChanges[typeof(T)] = value; } } @@ -146,14 +146,14 @@ public void AddToCache(T? serializedChanges = default) /// Adds a new entry to the cache. /// /// Cache type - /// Serialization changes + /// The data of the entry. /// The entry has already been cached. /// Note: This method does not read the internal storage to check whether the record already exists. - internal void AddToCache(Type type, object? serializedChanges) + internal void AddToCache(Type type, object? value) { lock (_dictionary) { - _serializedCacheChanges[type] = serializedChanges; + _serializedCacheChanges[type] = value; } } From 83d13ccb53d7521d0a5858ee3f95777dbcdbee29 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Tue, 21 Jan 2025 11:49:00 +0100 Subject: [PATCH 07/32] Clean code --- src/Neo/Persistence/DataCache.cs | 7 +++---- src/Neo/SmartContract/Native/NeoToken.cs | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index 97f6731c02..5d77f949c5 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -429,11 +429,10 @@ public bool Contains(StorageKey key) /// /// The key of the entry. /// Serialized cache - /// A delegate used to create the entry if it doesn't exist. If the entry already exists, the factory will not be used. - /// The cached data. Or if it doesn't exist and the is not provided. - public StorageItem? GetAndChange(StorageKey key, T serializedCache, Func? factory = null) + /// The cached data. + public StorageItem? GetAndChange(StorageKey key, T serializedCache) where T : IStorageCacheEntry { - var ret = GetAndChange(key, factory); + var ret = GetAndChange(key, serializedCache.GetStorageItem); _serializedCacheChanges[typeof(T)] = serializedCache; return ret; } diff --git a/src/Neo/SmartContract/Native/NeoToken.cs b/src/Neo/SmartContract/Native/NeoToken.cs index c0d4b4c33b..bc1a8d5ab4 100644 --- a/src/Neo/SmartContract/Native/NeoToken.cs +++ b/src/Neo/SmartContract/Native/NeoToken.cs @@ -284,7 +284,7 @@ private void SetGasPerBlock(ApplicationEngine engine, BigInteger gasPerBlock) uint index = engine.PersistingBlock.Index + 1; var entry = engine.SnapshotCache.GetAndChange(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(index), - new LastGasPerBlock() { GasPerBlock = gasPerBlock, Index = index }, () => new StorageItem(gasPerBlock)); + new LastGasPerBlock() { GasPerBlock = gasPerBlock, Index = index }); entry.Set(gasPerBlock); } From 5230d9c42d25e5394cc6b876395af5a24dbd9c26 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Tue, 21 Jan 2025 11:50:28 +0100 Subject: [PATCH 08/32] fix cache --- src/Neo/SmartContract/Native/NeoToken.cs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Neo/SmartContract/Native/NeoToken.cs b/src/Neo/SmartContract/Native/NeoToken.cs index bc1a8d5ab4..6e5ee2d088 100644 --- a/src/Neo/SmartContract/Native/NeoToken.cs +++ b/src/Neo/SmartContract/Native/NeoToken.cs @@ -304,13 +304,7 @@ public BigInteger GetGasPerBlock(DataCache snapshot) } var last = GetSortedGasRecords(snapshot, end).First(); - - if (cached == null) - { - // Not cached - snapshot.AddToCache(new LastGasPerBlock() { Index = last.Index, GasPerBlock = last.GasPerBlock }); - } - + snapshot.AddToCache(new LastGasPerBlock() { Index = last.Index, GasPerBlock = last.GasPerBlock }); return last.GasPerBlock; } From df7ad8e5e133ea597daf82c1b8208a32ea86c523 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Tue, 21 Jan 2025 11:58:02 +0100 Subject: [PATCH 09/32] remove lock --- src/Neo/Persistence/DataCache.cs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index 5d77f949c5..4e8da35764 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -136,10 +136,7 @@ public void Add(StorageKey key, T value) where T : IStorageCacheEntry /// Note: This method does not read the internal storage to check whether the record already exists. public void AddToCache(T? value = default) where T : IStorageCacheEntry { - lock (_dictionary) - { - _serializedCacheChanges[typeof(T)] = value; - } + _serializedCacheChanges[typeof(T)] = value; } /// @@ -151,10 +148,7 @@ public void AddToCache(T? value = default) where T : IStorageCacheEntry /// Note: This method does not read the internal storage to check whether the record already exists. internal void AddToCache(Type type, object? value) { - lock (_dictionary) - { - _serializedCacheChanges[type] = value; - } + _serializedCacheChanges[type] = value; } /// From c12767a37cb439c14b4fb1d411c320a58877e266 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Wed, 22 Jan 2025 09:56:11 +0100 Subject: [PATCH 10/32] Clean and Fix UT --- src/Neo/Persistence/ClonedCache.cs | 10 +--------- src/Neo/Persistence/DataCache.cs | 18 +++++++++++++++--- src/Neo/Persistence/SerializedCache.cs | 2 ++ src/Neo/Persistence/SnapshotCache.cs | 9 +-------- src/Neo/SmartContract/Native/NeoToken.cs | 18 +++++++----------- 5 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/Neo/Persistence/ClonedCache.cs b/src/Neo/Persistence/ClonedCache.cs index f3f7ffaf52..05204458b1 100644 --- a/src/Neo/Persistence/ClonedCache.cs +++ b/src/Neo/Persistence/ClonedCache.cs @@ -12,7 +12,6 @@ #nullable enable using Neo.SmartContract; -using System; using System.Collections.Generic; namespace Neo.Persistence @@ -21,18 +20,11 @@ class ClonedCache : DataCache { private readonly DataCache _innerCache; - public override SerializedCache SerializedCache => _innerCache.SerializedCache; - - public ClonedCache(DataCache innerCache) + public ClonedCache(DataCache innerCache) : base(innerCache.SerializedCache) { _innerCache = innerCache; } - protected override void SetCacheInternal(Type type, object? value) - { - _innerCache.AddToCache(type, value); - } - protected override void AddInternal(StorageKey key, StorageItem value) { _innerCache.Add(key, value.Clone()); diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index 4e8da35764..d090fe226f 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -54,7 +54,16 @@ public class Trackable(StorageKey key, StorageItem item, TrackState state) /// /// Serialized cache /// - public abstract SerializedCache SerializedCache { get; } + public SerializedCache SerializedCache { get; } + + /// + /// Constructor + /// + /// Serialized cache + protected DataCache(SerializedCache serializedCache) + { + SerializedCache = serializedCache; + } /// /// Reads a specified entry from the cache. If the entry is not in the cache, it will be automatically loaded from the underlying storage. @@ -186,7 +195,7 @@ public virtual void Commit() } foreach (var serialized in _serializedCacheChanges) { - SetCacheInternal(serialized.Key, serialized); + SetCacheInternal(serialized.Key, serialized.Value); } _serializedCacheChanges.Clear(); _changeSet.Clear(); @@ -254,7 +263,10 @@ public void Delete(StorageKey key) /// /// Type /// Value - protected abstract void SetCacheInternal(Type type, object? value); + protected void SetCacheInternal(Type type, object? value) + { + SerializedCache.Set(type, value); + } /// /// Finds the entries starting with the specified prefix. diff --git a/src/Neo/Persistence/SerializedCache.cs b/src/Neo/Persistence/SerializedCache.cs index 08710bc27c..e7370f31fa 100644 --- a/src/Neo/Persistence/SerializedCache.cs +++ b/src/Neo/Persistence/SerializedCache.cs @@ -14,6 +14,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Runtime.CompilerServices; namespace Neo.Persistence { @@ -41,6 +42,7 @@ public class SerializedCache /// /// Type /// Value + [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] public void Set(T? value) { Set(typeof(T), value); diff --git a/src/Neo/Persistence/SnapshotCache.cs b/src/Neo/Persistence/SnapshotCache.cs index 5609e59d2e..4a8686b56d 100644 --- a/src/Neo/Persistence/SnapshotCache.cs +++ b/src/Neo/Persistence/SnapshotCache.cs @@ -27,23 +27,16 @@ public class SnapshotCache : DataCache, IDisposable private readonly IReadOnlyStore _store; private readonly ISnapshot? _snapshot; - public override SerializedCache SerializedCache => _store.SerializedCache; - /// /// Initializes a new instance of the class. /// /// An to create a readonly cache; or an to create a snapshot cache. - public SnapshotCache(IReadOnlyStore store) + public SnapshotCache(IReadOnlyStore store) : base(store.SerializedCache) { _store = store; _snapshot = store as ISnapshot; } - protected override void SetCacheInternal(Type type, object? value) - { - _store.SerializedCache.Set(type, value); - } - protected override void AddInternal(StorageKey key, StorageItem value) { _snapshot?.Put(key.ToArray(), value.ToArray()); diff --git a/src/Neo/SmartContract/Native/NeoToken.cs b/src/Neo/SmartContract/Native/NeoToken.cs index 6e5ee2d088..016949de80 100644 --- a/src/Neo/SmartContract/Native/NeoToken.cs +++ b/src/Neo/SmartContract/Native/NeoToken.cs @@ -57,10 +57,10 @@ public sealed class NeoToken : FungibleToken private readonly StorageKey _votersCount; private readonly StorageKey _registerPrice; - private class LastGasPerBlock : IStorageCacheEntry + private class LastGasPerBlock(BigInteger gasPerBlock, long index) : IStorageCacheEntry { - public BigInteger GasPerBlock = 0; - public long Index = 0; + public readonly BigInteger GasPerBlock = gasPerBlock; + public readonly long Index = index; public StorageItem GetStorageItem() => new(GasPerBlock); } @@ -199,11 +199,7 @@ internal override ContractTask InitializeAsync(ApplicationEngine engine, Hardfor var cachedCommittee = new CachedCommittee(engine.ProtocolSettings.StandbyCommittee.Select(p => (p, BigInteger.Zero))); engine.SnapshotCache.Add(CreateStorageKey(Prefix_Committee), new StorageItem(cachedCommittee)); engine.SnapshotCache.Add(_votersCount, new StorageItem(System.Array.Empty())); - engine.SnapshotCache.Add(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(0u), new LastGasPerBlock() - { - GasPerBlock = 5 * GAS.Factor, - Index = 0 - }); + engine.SnapshotCache.Add(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(0u), new LastGasPerBlock(5 * GAS.Factor, 0)); engine.SnapshotCache.Add(_registerPrice, new StorageItem(1000 * GAS.Factor)); return Mint(engine, Contract.GetBFTAddress(engine.ProtocolSettings.StandbyValidators), TotalAmount, false); } @@ -282,9 +278,9 @@ private void SetGasPerBlock(ApplicationEngine engine, BigInteger gasPerBlock) throw new ArgumentOutOfRangeException(nameof(gasPerBlock)); if (!CheckCommittee(engine)) throw new InvalidOperationException(); - uint index = engine.PersistingBlock.Index + 1; + var index = engine.PersistingBlock.Index + 1; var entry = engine.SnapshotCache.GetAndChange(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(index), - new LastGasPerBlock() { GasPerBlock = gasPerBlock, Index = index }); + new LastGasPerBlock(gasPerBlock, index)); entry.Set(gasPerBlock); } @@ -304,7 +300,7 @@ public BigInteger GetGasPerBlock(DataCache snapshot) } var last = GetSortedGasRecords(snapshot, end).First(); - snapshot.AddToCache(new LastGasPerBlock() { Index = last.Index, GasPerBlock = last.GasPerBlock }); + snapshot.AddToCache(new LastGasPerBlock(last.GasPerBlock, last.Index)); return last.GasPerBlock; } From 601c3667609d5ab2edf79b14cf861c3ed0f864c3 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Wed, 22 Jan 2025 09:58:56 +0100 Subject: [PATCH 11/32] Clean --- src/Neo/Persistence/DataCache.cs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index d090fe226f..70a4d0feaf 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -13,6 +13,7 @@ using Neo.Extensions; using Neo.SmartContract; +using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; @@ -195,7 +196,7 @@ public virtual void Commit() } foreach (var serialized in _serializedCacheChanges) { - SetCacheInternal(serialized.Key, serialized.Value); + SerializedCache.Set(serialized.Key, serialized.Value); } _serializedCacheChanges.Clear(); _changeSet.Clear(); @@ -258,16 +259,6 @@ public void Delete(StorageKey key) /// The key of the entry. protected abstract void DeleteInternal(StorageKey key); - /// - /// Set cache internal - /// - /// Type - /// Value - protected void SetCacheInternal(Type type, object? value) - { - SerializedCache.Set(type, value); - } - /// /// Finds the entries starting with the specified prefix. /// From 6bdcc21a67be1ddc5dcd3b94cc30dc512ba23c6f Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Thu, 23 Jan 2025 09:16:54 +0100 Subject: [PATCH 12/32] Anna's feedback --- src/Neo/Persistence/DataCache.cs | 2 +- src/Neo/SmartContract/Native/NeoToken.cs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index 70a4d0feaf..7f594656df 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -134,7 +134,7 @@ public void Add(StorageKey key, T value) where T : IStorageCacheEntry lock (_dictionary) { Add(key, value.GetStorageItem()); - _serializedCacheChanges[typeof(T)] = value; + AddToCache(value); } } diff --git a/src/Neo/SmartContract/Native/NeoToken.cs b/src/Neo/SmartContract/Native/NeoToken.cs index 016949de80..c4597db14d 100644 --- a/src/Neo/SmartContract/Native/NeoToken.cs +++ b/src/Neo/SmartContract/Native/NeoToken.cs @@ -196,10 +196,11 @@ internal override ContractTask InitializeAsync(ApplicationEngine engine, Hardfor { if (hardfork == ActiveIn) { + var initIndex = engine.PersistingBlock?.Index ?? 0u; var cachedCommittee = new CachedCommittee(engine.ProtocolSettings.StandbyCommittee.Select(p => (p, BigInteger.Zero))); engine.SnapshotCache.Add(CreateStorageKey(Prefix_Committee), new StorageItem(cachedCommittee)); engine.SnapshotCache.Add(_votersCount, new StorageItem(System.Array.Empty())); - engine.SnapshotCache.Add(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(0u), new LastGasPerBlock(5 * GAS.Factor, 0)); + engine.SnapshotCache.Add(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(initIndex), new LastGasPerBlock(5 * GAS.Factor, initIndex)); engine.SnapshotCache.Add(_registerPrice, new StorageItem(1000 * GAS.Factor)); return Mint(engine, Contract.GetBFTAddress(engine.ProtocolSettings.StandbyValidators), TotalAmount, false); } From 5fb4b62f73856fb69514c3df4169ecef0d2a6f3d Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Thu, 23 Jan 2025 09:31:26 +0100 Subject: [PATCH 13/32] Add fee values --- src/Neo/Persistence/DataCache.cs | 5 +- src/Neo/SmartContract/Native/NeoToken.cs | 4 +- .../SmartContract/Native/PolicyContract.cs | 58 +++++++++++++++---- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index 7f594656df..28878fe7b0 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -426,12 +426,11 @@ public bool Contains(StorageKey key) /// /// The key of the entry. /// Serialized cache - /// The cached data. - public StorageItem? GetAndChange(StorageKey key, T serializedCache) where T : IStorageCacheEntry + public void GetAndChange(StorageKey key, T serializedCache) where T : IStorageCacheEntry { var ret = GetAndChange(key, serializedCache.GetStorageItem); + ret!.FromReplica(serializedCache.GetStorageItem()); _serializedCacheChanges[typeof(T)] = serializedCache; - return ret; } /// diff --git a/src/Neo/SmartContract/Native/NeoToken.cs b/src/Neo/SmartContract/Native/NeoToken.cs index c4597db14d..3838e777f4 100644 --- a/src/Neo/SmartContract/Native/NeoToken.cs +++ b/src/Neo/SmartContract/Native/NeoToken.cs @@ -280,9 +280,7 @@ private void SetGasPerBlock(ApplicationEngine engine, BigInteger gasPerBlock) if (!CheckCommittee(engine)) throw new InvalidOperationException(); var index = engine.PersistingBlock.Index + 1; - var entry = engine.SnapshotCache.GetAndChange(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(index), - new LastGasPerBlock(gasPerBlock, index)); - entry.Set(gasPerBlock); + engine.SnapshotCache.GetAndChange(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(index), new LastGasPerBlock(gasPerBlock, index)); } /// diff --git a/src/Neo/SmartContract/Native/PolicyContract.cs b/src/Neo/SmartContract/Native/PolicyContract.cs index d034c06b46..1450dc7e55 100644 --- a/src/Neo/SmartContract/Native/PolicyContract.cs +++ b/src/Neo/SmartContract/Native/PolicyContract.cs @@ -69,6 +69,23 @@ public sealed class PolicyContract : NativeContract private readonly StorageKey _execFeeFactor; private readonly StorageKey _storagePrice; + private class LastFeePerByte(long feePerByte) : IStorageCacheEntry + { + public readonly long FeePerByte = feePerByte; + public StorageItem GetStorageItem() => new(FeePerByte); + } + + private class LastStorageFee(uint storagePrice) : IStorageCacheEntry + { + public readonly uint StoragePrice = storagePrice; + public StorageItem GetStorageItem() => new(StoragePrice); + } + + private class LastExecFee(uint execFeeFactor) : IStorageCacheEntry + { + public readonly uint ExecFeeFactor = execFeeFactor; + public StorageItem GetStorageItem() => new(ExecFeeFactor); + } internal PolicyContract() : base() { @@ -81,9 +98,9 @@ internal override ContractTask InitializeAsync(ApplicationEngine engine, Hardfor { if (hardfork == ActiveIn) { - engine.SnapshotCache.Add(_feePerByte, new StorageItem(DefaultFeePerByte)); - engine.SnapshotCache.Add(_execFeeFactor, new StorageItem(DefaultExecFeeFactor)); - engine.SnapshotCache.Add(_storagePrice, new StorageItem(DefaultStoragePrice)); + engine.SnapshotCache.Add(_feePerByte, new LastFeePerByte(DefaultFeePerByte)); + engine.SnapshotCache.Add(_execFeeFactor, new LastExecFee(DefaultExecFeeFactor)); + engine.SnapshotCache.Add(_storagePrice, new LastStorageFee(DefaultStoragePrice)); } return ContractTask.CompletedTask; } @@ -96,7 +113,14 @@ internal override ContractTask InitializeAsync(ApplicationEngine engine, Hardfor [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.ReadStates)] public long GetFeePerByte(DataCache snapshot) { - return (long)(BigInteger)snapshot[_feePerByte]; + var cached = snapshot.SerializedCache.Get(); + if (cached != null) + { + return cached.FeePerByte; + } + var fee = (long)(BigInteger)snapshot[_feePerByte]; + snapshot.AddToCache(new LastFeePerByte(fee)); + return fee; } /// @@ -107,7 +131,14 @@ public long GetFeePerByte(DataCache snapshot) [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.ReadStates)] public uint GetExecFeeFactor(DataCache snapshot) { - return (uint)(BigInteger)snapshot[_execFeeFactor]; + var cached = snapshot.SerializedCache.Get(); + if (cached != null) + { + return cached.ExecFeeFactor; + } + var fee = (uint)(BigInteger)snapshot[_execFeeFactor]; + snapshot.AddToCache(new LastExecFee(fee)); + return fee; } /// @@ -118,7 +149,14 @@ public uint GetExecFeeFactor(DataCache snapshot) [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.ReadStates)] public uint GetStoragePrice(DataCache snapshot) { - return (uint)(BigInteger)snapshot[_storagePrice]; + var cached = snapshot.SerializedCache.Get(); + if (cached != null) + { + return cached.StoragePrice; + } + var fee = (uint)(BigInteger)snapshot[_storagePrice]; + snapshot.AddToCache(new LastStorageFee(fee)); + return fee; } /// @@ -131,7 +169,7 @@ public uint GetStoragePrice(DataCache snapshot) public uint GetAttributeFee(DataCache snapshot, byte attributeType) { if (!Enum.IsDefined(typeof(TransactionAttributeType), attributeType)) throw new InvalidOperationException(); - StorageItem entry = snapshot.TryGet(CreateStorageKey(Prefix_AttributeFee).Add(attributeType)); + var entry = snapshot.TryGet(CreateStorageKey(Prefix_AttributeFee).Add(attributeType)); if (entry == null) return DefaultAttributeFee; return (uint)(BigInteger)entry; @@ -164,7 +202,7 @@ private void SetFeePerByte(ApplicationEngine engine, long value) { if (value < 0 || value > 1_00000000) throw new ArgumentOutOfRangeException(nameof(value)); if (!CheckCommittee(engine)) throw new InvalidOperationException(); - engine.SnapshotCache.GetAndChange(_feePerByte).Set(value); + engine.SnapshotCache.GetAndChange(_feePerByte, new LastFeePerByte(value)); } [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.States)] @@ -172,7 +210,7 @@ private void SetExecFeeFactor(ApplicationEngine engine, uint value) { if (value == 0 || value > MaxExecFeeFactor) throw new ArgumentOutOfRangeException(nameof(value)); if (!CheckCommittee(engine)) throw new InvalidOperationException(); - engine.SnapshotCache.GetAndChange(_execFeeFactor).Set(value); + engine.SnapshotCache.GetAndChange(_execFeeFactor, new LastExecFee(value)); } [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.States)] @@ -180,7 +218,7 @@ private void SetStoragePrice(ApplicationEngine engine, uint value) { if (value == 0 || value > MaxStoragePrice) throw new ArgumentOutOfRangeException(nameof(value)); if (!CheckCommittee(engine)) throw new InvalidOperationException(); - engine.SnapshotCache.GetAndChange(_storagePrice).Set(value); + engine.SnapshotCache.GetAndChange(_storagePrice, new LastStorageFee(value)); } [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.States)] From 9decaf0ea68e676d7ffa016d8a06aeb0359d4250 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Thu, 6 Feb 2025 10:39:53 +0100 Subject: [PATCH 14/32] ``` | Method | Mean | Error | StdDev | |------------- |----------:|----------:|----------:| | WithCache | 40.66 us | 0.130 us | 0.115 us | | WithoutCache | 810.69 us | 15.792 us | 13.999 us | ``` --- benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs | 69 +++++++++++++++++++ benchmarks/Neo.Benchmarks/Program.cs | 14 ++-- src/Neo/Neo.csproj | 1 + src/Neo/Persistence/DataCache.cs | 29 ++++++-- src/Neo/Persistence/SerializedCache.cs | 22 +++++- .../SmartContract/Native/HashIndexState.cs | 2 +- .../SmartContract/Native/LedgerContract.cs | 1 - src/Neo/SmartContract/Native/NeoToken.cs | 8 +-- .../SmartContract/Native/PolicyContract.cs | 6 +- 9 files changed, 126 insertions(+), 26 deletions(-) create mode 100644 benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs diff --git a/benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs b/benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs new file mode 100644 index 0000000000..7da9b26f31 --- /dev/null +++ b/benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs @@ -0,0 +1,69 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// Benchmarks.Cache.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using BenchmarkDotNet.Attributes; +using Neo.Persistence; +using Neo.SmartContract; +using Neo.SmartContract.Native; +using System.Numerics; + +namespace Neo.Benchmark +{ + public class Benchmarks_Cache + { + // 256 KiB + readonly MemoryStore _store; + readonly SnapshotCache _snapshot; + + public Benchmarks_Cache() + { + _store = new MemoryStore(); + _snapshot = new(_store.GetSnapshot()); + + // Ledger.CurrentIndex + + _snapshot.GetAndChange(new KeyBuilder(NativeContract.Ledger.Id, 12), () => new StorageItem(new HashIndexState() { Hash = UInt256.Zero, Index = 2 })); + + // Gas Per block + + _snapshot.GetAndChange(new KeyBuilder(NativeContract.NEO.Id, 29).AddBigEndian(0), () => new StorageItem(0)); + _snapshot.GetAndChange(new KeyBuilder(NativeContract.NEO.Id, 29).AddBigEndian(1), () => new StorageItem(1)); + _snapshot.GetAndChange(new KeyBuilder(NativeContract.NEO.Id, 29).AddBigEndian(2), () => new StorageItem(2)); + } + + [Benchmark] + public void WithCache() + { + for (var x = 0; x < 1_000; x++) + { + var ret = NativeContract.NEO.GetGasPerBlock(_snapshot); + if (ret != 2) throw new Exception("Test error"); + } + } + + [Benchmark] + public void WithoutCache() + { + for (var x = 0; x < 1_000; x++) + { + var ret = OldCode(); + if (ret != 2) throw new Exception("Test error"); + } + } + + private BigInteger OldCode() + { + var end = NativeContract.Ledger.CurrentIndex(_snapshot) + 1; + var last = NativeContract.NEO.GetSortedGasRecords(_snapshot, end).First(); + return last.GasPerBlock; + } + } +} diff --git a/benchmarks/Neo.Benchmarks/Program.cs b/benchmarks/Neo.Benchmarks/Program.cs index 437da9ed94..f3f63c5d9c 100644 --- a/benchmarks/Neo.Benchmarks/Program.cs +++ b/benchmarks/Neo.Benchmarks/Program.cs @@ -11,12 +11,10 @@ using BenchmarkDotNet.Running; using Neo.Benchmark; -using Neo.Benchmarks.Persistence.Benchmarks; -using Neo.SmartContract.Benchmark; -// BenchmarkRunner.Run(); -BenchmarkRunner.Run(); -BenchmarkRunner.Run(); -BenchmarkRunner.Run(); -BenchmarkRunner.Run(); -BenchmarkRunner.Run(); +BenchmarkRunner.Run(); +//BenchmarkRunner.Run(); +//BenchmarkRunner.Run(); +//BenchmarkRunner.Run(); +//BenchmarkRunner.Run(); +//BenchmarkRunner.Run(); diff --git a/src/Neo/Neo.csproj b/src/Neo/Neo.csproj index 32af25af36..c098e9f63a 100644 --- a/src/Neo/Neo.csproj +++ b/src/Neo/Neo.csproj @@ -29,6 +29,7 @@ + diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index 28878fe7b0..d4db99b573 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -48,9 +48,9 @@ public class Trackable(StorageKey key, StorageItem item, TrackState state) public TrackState State { get; set; } = state; } - private readonly Dictionary _dictionary = new(); - private readonly HashSet _changeSet = new(); - private readonly Dictionary _serializedCacheChanges = new(); + private readonly Dictionary _dictionary = []; + private readonly HashSet _changeSet = []; + private readonly Dictionary _serializedCacheChanges = []; /// /// Serialized cache @@ -146,7 +146,7 @@ public void Add(StorageKey key, T value) where T : IStorageCacheEntry /// Note: This method does not read the internal storage to check whether the record already exists. public void AddToCache(T? value = default) where T : IStorageCacheEntry { - _serializedCacheChanges[typeof(T)] = value; + AddToCache(typeof(T), value); } /// @@ -156,9 +156,26 @@ public void AddToCache(T? value = default) where T : IStorageCacheEntry /// The data of the entry. /// The entry has already been cached. /// Note: This method does not read the internal storage to check whether the record already exists. - internal void AddToCache(Type type, object? value) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void AddToCache(Type type, IStorageCacheEntry? value) { _serializedCacheChanges[type] = value; + SerializedCache.Remove(type); + } + + /// + /// Get from cache + /// + /// Cache type + /// Entry + public T? GetFromCache() where T : IStorageCacheEntry + { + if (_serializedCacheChanges.TryGetValue(typeof(T), out var value)) + { + return (T?)value; + } + + return SerializedCache.Get(); } /// @@ -430,7 +447,7 @@ public void GetAndChange(StorageKey key, T serializedCache) where T : IStorag { var ret = GetAndChange(key, serializedCache.GetStorageItem); ret!.FromReplica(serializedCache.GetStorageItem()); - _serializedCacheChanges[typeof(T)] = serializedCache; + AddToCache(serializedCache); } /// diff --git a/src/Neo/Persistence/SerializedCache.cs b/src/Neo/Persistence/SerializedCache.cs index e7370f31fa..9fde072090 100644 --- a/src/Neo/Persistence/SerializedCache.cs +++ b/src/Neo/Persistence/SerializedCache.cs @@ -42,7 +42,7 @@ public class SerializedCache /// /// Type /// Value - [MethodImplAttribute(MethodImplOptions.AggressiveInlining)] + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Set(T? value) { Set(typeof(T), value); @@ -55,8 +55,24 @@ public void Set(T? value) /// Value public void Set(Type type, object? value) { - if (value == null) _cache.Remove(type, out _); - else _cache[type] = value; + if (value == null) + { + Remove(type); + } + else + { + _cache[type] = value; + } + } + + /// + /// Remove entry + /// + /// Type + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Remove(Type type) + { + _cache.Remove(type, out _); } } } diff --git a/src/Neo/SmartContract/Native/HashIndexState.cs b/src/Neo/SmartContract/Native/HashIndexState.cs index 06bc5704cb..7ceeeee72e 100644 --- a/src/Neo/SmartContract/Native/HashIndexState.cs +++ b/src/Neo/SmartContract/Native/HashIndexState.cs @@ -15,7 +15,7 @@ namespace Neo.SmartContract.Native { - class HashIndexState : IInteroperable + internal class HashIndexState : IInteroperable { public UInt256 Hash; public uint Index; diff --git a/src/Neo/SmartContract/Native/LedgerContract.cs b/src/Neo/SmartContract/Native/LedgerContract.cs index 88e9243dd1..e7deb6261f 100644 --- a/src/Neo/SmartContract/Native/LedgerContract.cs +++ b/src/Neo/SmartContract/Native/LedgerContract.cs @@ -12,7 +12,6 @@ #pragma warning disable IDE0051 using Neo.Extensions; -using Neo.IO; using Neo.Network.P2P.Payloads; using Neo.Persistence; using Neo.VM; diff --git a/src/Neo/SmartContract/Native/NeoToken.cs b/src/Neo/SmartContract/Native/NeoToken.cs index 748b5048a8..26e5b87b33 100644 --- a/src/Neo/SmartContract/Native/NeoToken.cs +++ b/src/Neo/SmartContract/Native/NeoToken.cs @@ -305,7 +305,7 @@ private void SetGasPerBlock(ApplicationEngine engine, BigInteger gasPerBlock) public BigInteger GetGasPerBlock(DataCache snapshot) { var end = Ledger.CurrentIndex(snapshot) + 1; - var cached = snapshot.SerializedCache.Get(); + var cached = snapshot.GetFromCache(); if (cached != null && cached.Index < end) { return cached.GasPerBlock; @@ -337,10 +337,10 @@ public long GetRegisterPrice(DataCache snapshot) return (long)(BigInteger)snapshot[_registerPrice]; } - private IEnumerable<(uint Index, BigInteger GasPerBlock)> GetSortedGasRecords(DataCache snapshot, uint end) + internal IEnumerable<(uint Index, BigInteger GasPerBlock)> GetSortedGasRecords(DataCache snapshot, uint end) { - byte[] key = CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(end).ToArray(); - byte[] boundary = CreateStorageKey(Prefix_GasPerBlock).ToArray(); + var key = CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(end).ToArray(); + var boundary = CreateStorageKey(Prefix_GasPerBlock).ToArray(); return snapshot.FindRange(key, boundary, SeekDirection.Backward) .Select(u => (BinaryPrimitives.ReadUInt32BigEndian(u.Key.Key.Span[^sizeof(uint)..]), (BigInteger)u.Value)); } diff --git a/src/Neo/SmartContract/Native/PolicyContract.cs b/src/Neo/SmartContract/Native/PolicyContract.cs index 1450dc7e55..cab0b36587 100644 --- a/src/Neo/SmartContract/Native/PolicyContract.cs +++ b/src/Neo/SmartContract/Native/PolicyContract.cs @@ -113,7 +113,7 @@ internal override ContractTask InitializeAsync(ApplicationEngine engine, Hardfor [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.ReadStates)] public long GetFeePerByte(DataCache snapshot) { - var cached = snapshot.SerializedCache.Get(); + var cached = snapshot.GetFromCache(); if (cached != null) { return cached.FeePerByte; @@ -131,7 +131,7 @@ public long GetFeePerByte(DataCache snapshot) [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.ReadStates)] public uint GetExecFeeFactor(DataCache snapshot) { - var cached = snapshot.SerializedCache.Get(); + var cached = snapshot.GetFromCache(); if (cached != null) { return cached.ExecFeeFactor; @@ -149,7 +149,7 @@ public uint GetExecFeeFactor(DataCache snapshot) [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.ReadStates)] public uint GetStoragePrice(DataCache snapshot) { - var cached = snapshot.SerializedCache.Get(); + var cached = snapshot.GetFromCache(); if (cached != null) { return cached.StoragePrice; From a84c163db056c0ad5a77fa2c54af9375d77e082a Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Thu, 6 Feb 2025 10:45:14 +0100 Subject: [PATCH 15/32] Fix conflicts --- src/Neo/Persistence/DataCache.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index 3d1571e4bc..7b1d957d9d 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -13,7 +13,6 @@ using Neo.Extensions; using Neo.SmartContract; -using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; @@ -448,7 +447,8 @@ public bool Contains(StorageKey key) } /// - /// Reads a specified entry from the cache, and mark it as . If the entry is not in the cache, it will be automatically loaded from the underlying storage. + /// Reads a specified entry from the cache, and mark it as . + /// If the entry is not in the cache, it will be automatically loaded from the underlying storage. /// /// The key of the entry. /// Serialized cache @@ -460,7 +460,9 @@ public void GetAndChange(StorageKey key, T serializedCache) where T : IStorag } /// - /// Reads a specified entry from the cache. If the entry is not in the cache, it will be automatically loaded from the underlying storage. If the entry doesn't exist, the factory will be used to create a new one. + /// Reads a specified entry from the cache. + /// If the entry is not in the cache, it will be automatically loaded from the underlying storage. + /// If the entry doesn't exist, the factory will be used to create a new one. /// /// The key of the entry. /// From 3c6d29247fd591958b2f5d3262d157194930c60b Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Thu, 6 Feb 2025 10:52:56 +0100 Subject: [PATCH 16/32] Optimize --- src/Neo/Persistence/DataCache.cs | 15 +++++----- src/Neo/Persistence/SerializedCache.cs | 41 ++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index 7b1d957d9d..c3bf508f12 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -49,7 +49,7 @@ public class Trackable(StorageKey key, StorageItem item, TrackState state) private readonly Dictionary _dictionary = []; private readonly HashSet _changeSet = []; - private readonly Dictionary _serializedCacheChanges = []; + private readonly SerializedCache _serializedCacheChanges = new(); /// /// Serialized cache @@ -158,7 +158,7 @@ public void AddToCache(T? value = default) where T : IStorageCacheEntry [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void AddToCache(Type type, IStorageCacheEntry? value) { - _serializedCacheChanges[type] = value; + _serializedCacheChanges.Set(type, value); SerializedCache.Remove(type); } @@ -169,9 +169,11 @@ internal void AddToCache(Type type, IStorageCacheEntry? value) /// Entry public T? GetFromCache() where T : IStorageCacheEntry { - if (_serializedCacheChanges.TryGetValue(typeof(T), out var value)) + var value = _serializedCacheChanges.Get(); + + if (value != null) { - return (T?)value; + return value; } return SerializedCache.Get(); @@ -210,10 +212,7 @@ public virtual void Commit() break; } } - foreach (var serialized in _serializedCacheChanges) - { - SerializedCache.Set(serialized.Key, serialized.Value); - } + SerializedCache.CopyFrom(_serializedCacheChanges); _serializedCacheChanges.Clear(); _changeSet.Clear(); } diff --git a/src/Neo/Persistence/SerializedCache.cs b/src/Neo/Persistence/SerializedCache.cs index 9fde072090..588bd827b5 100644 --- a/src/Neo/Persistence/SerializedCache.cs +++ b/src/Neo/Persistence/SerializedCache.cs @@ -12,7 +12,6 @@ #nullable enable using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Runtime.CompilerServices; @@ -20,7 +19,7 @@ namespace Neo.Persistence { public class SerializedCache { - private readonly ConcurrentDictionary _cache = new(); + private readonly Dictionary _cache = []; /// /// Get cached entry @@ -43,7 +42,7 @@ public class SerializedCache /// Type /// Value [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Set(T? value) + public void Set(T? value) where T : IStorageCacheEntry { Set(typeof(T), value); } @@ -53,7 +52,7 @@ public void Set(T? value) /// /// Type /// Value - public void Set(Type type, object? value) + public void Set(Type type, IStorageCacheEntry? value) { if (value == null) { @@ -61,7 +60,10 @@ public void Set(Type type, object? value) } else { - _cache[type] = value; + lock (_cache) + { + _cache[type] = value; + } } } @@ -72,7 +74,34 @@ public void Set(Type type, object? value) [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Remove(Type type) { - _cache.Remove(type, out _); + lock (_cache) + { + _cache.Remove(type, out _); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Clear() + { + lock (_cache) + { + _cache.Clear(); + } + } + + /// + /// Copy from + /// + /// Value + public void CopyFrom(SerializedCache value) + { + lock (_cache) lock (value._cache) + { + foreach (var serialized in value._cache) + { + _cache[serialized.Key] = serialized.Value; + } + } } } } From bf97fabbe4d9ad66fb7e4d6168f322bb9950f95f Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Thu, 6 Feb 2025 11:02:35 +0100 Subject: [PATCH 17/32] Fix --- src/Neo/Persistence/DataCache.cs | 4 +-- src/Neo/Persistence/IReadOnlyStoreView.cs | 12 +++++++- src/Neo/Persistence/ReadOnlyStoreView.cs | 28 ++++++++++++++----- .../SmartContract/Native/PolicyContract.cs | 6 ++-- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index c3bf508f12..c2e5c7bacf 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -163,10 +163,10 @@ internal void AddToCache(Type type, IStorageCacheEntry? value) } /// - /// Get from cache + /// Tries to get the entry from cache. /// /// Cache type - /// Entry + /// The entry if found, null otherwise. public T? GetFromCache() where T : IStorageCacheEntry { var value = _serializedCacheChanges.Get(); diff --git a/src/Neo/Persistence/IReadOnlyStoreView.cs b/src/Neo/Persistence/IReadOnlyStoreView.cs index b5d4d050e7..54287721bd 100644 --- a/src/Neo/Persistence/IReadOnlyStoreView.cs +++ b/src/Neo/Persistence/IReadOnlyStoreView.cs @@ -9,8 +9,11 @@ // Redistribution and use in source and binary forms with or without // modifications are permitted. +#nullable enable + using Neo.SmartContract; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; namespace Neo.Persistence { @@ -35,12 +38,19 @@ public interface IReadOnlyStoreView /// The entry if found, throws a otherwise. StorageItem this[StorageKey key] { get; } + /// + /// Tries to get the entry from cache. + /// + /// Cache type + /// The entry if found, null otherwise. + T? GetFromCache() where T : IStorageCacheEntry; + /// /// Tries to get the entry with the specified key. /// /// The key to get. /// The entry if found, null otherwise. /// True if the entry exists, false otherwise. - bool TryGet(StorageKey key, out StorageItem item); + bool TryGet(StorageKey key, [NotNullWhen(true)] out StorageItem? item); } } diff --git a/src/Neo/Persistence/ReadOnlyStoreView.cs b/src/Neo/Persistence/ReadOnlyStoreView.cs index bd37d77719..baca6168c5 100644 --- a/src/Neo/Persistence/ReadOnlyStoreView.cs +++ b/src/Neo/Persistence/ReadOnlyStoreView.cs @@ -9,22 +9,25 @@ // Redistribution and use in source and binary forms with or without // modifications are permitted. +#nullable enable + using Neo.SmartContract; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; namespace Neo.Persistence { public class ReadOnlyStoreView : IReadOnlyStoreView { - private readonly IReadOnlyStore store; + private readonly IReadOnlyStore _store; public ReadOnlyStoreView(IReadOnlyStore store) { - this.store = store; + _store = store; } /// - public bool Contains(StorageKey key) => store.Contains(key.ToArray()); + public bool Contains(StorageKey key) => _store.Contains(key.ToArray()); /// public StorageItem this[StorageKey key] @@ -37,12 +40,23 @@ public StorageItem this[StorageKey key] } } + public T? GetFromCache() where T : IStorageCacheEntry + { + return _store.SerializedCache.Get(); + } + + /// - public bool TryGet(StorageKey key, out StorageItem item) + public bool TryGet(StorageKey key, [NotNullWhen(true)] out StorageItem? item) { - var ok = store.TryGet(key.ToArray(), out byte[] value); - item = ok ? new StorageItem(value) : null; - return ok; + if (_store.TryGet(key.ToArray(), out var value)) + { + item = new StorageItem(value); + return true; + } + + item = null; + return false; } } } diff --git a/src/Neo/SmartContract/Native/PolicyContract.cs b/src/Neo/SmartContract/Native/PolicyContract.cs index 895bba63d0..667e5bef69 100644 --- a/src/Neo/SmartContract/Native/PolicyContract.cs +++ b/src/Neo/SmartContract/Native/PolicyContract.cs @@ -111,7 +111,7 @@ internal override ContractTask InitializeAsync(ApplicationEngine engine, Hardfor /// The snapshot used to read data. /// The network fee per transaction byte. [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.ReadStates)] - public long GetFeePerByte(IReadOnlyStoreView snapshot) + public long GetFeePerByte(DataCache snapshot) { var cached = snapshot.GetFromCache(); if (cached != null) @@ -129,7 +129,7 @@ public long GetFeePerByte(IReadOnlyStoreView snapshot) /// The snapshot used to read data. /// The execution fee factor. [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.ReadStates)] - public uint GetExecFeeFactor(IReadOnlyStoreView snapshot) + public uint GetExecFeeFactor(DataCache snapshot) { var cached = snapshot.GetFromCache(); if (cached != null) @@ -147,7 +147,7 @@ public uint GetExecFeeFactor(IReadOnlyStoreView snapshot) /// The snapshot used to read data. /// The storage price. [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.ReadStates)] - public uint GetStoragePrice(IReadOnlyStoreView snapshot) + public uint GetStoragePrice(DataCache snapshot) { var cached = snapshot.GetFromCache(); if (cached != null) From 62ae4103f5c8adfd68ef8238cb08c52a0a5cfe5e Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Thu, 6 Feb 2025 11:04:27 +0100 Subject: [PATCH 18/32] Clean changes --- src/Neo/Persistence/IReadOnlyStoreView.cs | 7 ------- src/Neo/Persistence/ReadOnlyStoreView.cs | 6 ------ 2 files changed, 13 deletions(-) diff --git a/src/Neo/Persistence/IReadOnlyStoreView.cs b/src/Neo/Persistence/IReadOnlyStoreView.cs index 54287721bd..118f46abc7 100644 --- a/src/Neo/Persistence/IReadOnlyStoreView.cs +++ b/src/Neo/Persistence/IReadOnlyStoreView.cs @@ -38,13 +38,6 @@ public interface IReadOnlyStoreView /// The entry if found, throws a otherwise. StorageItem this[StorageKey key] { get; } - /// - /// Tries to get the entry from cache. - /// - /// Cache type - /// The entry if found, null otherwise. - T? GetFromCache() where T : IStorageCacheEntry; - /// /// Tries to get the entry with the specified key. /// diff --git a/src/Neo/Persistence/ReadOnlyStoreView.cs b/src/Neo/Persistence/ReadOnlyStoreView.cs index baca6168c5..d6ba6e17c1 100644 --- a/src/Neo/Persistence/ReadOnlyStoreView.cs +++ b/src/Neo/Persistence/ReadOnlyStoreView.cs @@ -40,12 +40,6 @@ public StorageItem this[StorageKey key] } } - public T? GetFromCache() where T : IStorageCacheEntry - { - return _store.SerializedCache.Get(); - } - - /// public bool TryGet(StorageKey key, [NotNullWhen(true)] out StorageItem? item) { From a21526f2ba554cae507b0743d5a77a581963e217 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Fri, 7 Feb 2025 00:08:34 +0800 Subject: [PATCH 19/32] add UT --- .../Persistence/UT_SerializedCache.cs | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 tests/Neo.UnitTests/Persistence/UT_SerializedCache.cs diff --git a/tests/Neo.UnitTests/Persistence/UT_SerializedCache.cs b/tests/Neo.UnitTests/Persistence/UT_SerializedCache.cs new file mode 100644 index 0000000000..5fb567ef0c --- /dev/null +++ b/tests/Neo.UnitTests/Persistence/UT_SerializedCache.cs @@ -0,0 +1,198 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// UT_SerializedCache.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Neo.Persistence; +using Neo.SmartContract; +using System; + +namespace Neo.UnitTests.Persistence +{ + // Dummy implementations for IStorageCacheEntry for testing purposes + public class TestCacheEntry(int value) : IStorageCacheEntry + { + public int Value { get; } = value; + public StorageItem GetStorageItem() => new() { Value = BitConverter.GetBytes(Value) }; + } + + public class TestCacheEntry2(string text) : IStorageCacheEntry + { + public string Text { get; } = text; + public StorageItem GetStorageItem() => new() { Value = System.Text.Encoding.UTF8.GetBytes(Text) }; + } + + [TestClass] + [TestCategory("SerializedCache")] + public class UT_SerializedCache + { + [TestMethod] + public void TestGetReturnsDefaultWhenNotSet() + { + var cache = new SerializedCache(); + Assert.IsNull(cache.Get(), "Expected null when cache does not contain the type"); + } + + [TestMethod] + public void TestSetAndGet() + { + var entry = new TestCacheEntry(42); + var cache = new SerializedCache(); + cache.Set(entry); + var retrieved = cache.Get(); + Assert.IsNotNull(retrieved, "Entry was not set correctly"); + Assert.AreEqual(42, retrieved.Value, "Retrieved entry does not match the set value"); + } + + [TestMethod] + public void TestRemove() + { + var entry = new TestCacheEntry(42); + var cache = new SerializedCache(); + cache.Set(entry); + cache.Remove(typeof(TestCacheEntry)); + Assert.IsNull(cache.Get(), "Entry should be null after removal"); + } + + [TestMethod] + public void TestClear() + { + var cache = new SerializedCache(); + cache.Set(new TestCacheEntry(1)); + cache.Set(new TestCacheEntry2("one")); + cache.Clear(); + Assert.IsNull(cache.Get(), "Cache should be cleared for first type"); + Assert.IsNull(cache.Get(), "Cache should be cleared for second type"); + } + + [TestMethod] + public void TestCopyFrom() + { + var source = new SerializedCache(); + source.Set(new TestCacheEntry(99)); + + var target = new SerializedCache(); + target.Set(new TestCacheEntry2("hello")); + + target.CopyFrom(source); + + var entryA = target.Get(); + var entryB = target.Get(); + + Assert.IsNotNull(entryA, "Copied entry should exist in target cache"); + Assert.AreEqual(99, entryA.Value, "Copied entry value mismatch"); + Assert.IsNotNull(entryB, "Existing entry in target cache should not be overwritten if types differ"); + Assert.AreEqual("hello", entryB.Text, "Existing entry text mismatch"); + } + + [TestMethod] + public void TestSetNullValueRemovesEntry() + { + var cache = new SerializedCache(); + var entry = new TestCacheEntry(42); + cache.Set(entry); + Assert.IsNotNull(cache.Get(), "Entry should be set initially"); + + cache.Set(null); + Assert.IsNull(cache.Get(), "Entry should be removed after setting null"); + } + + [TestMethod] + public void TestMultipleTypesStorage() + { + var cache = new SerializedCache(); + var entry1 = new TestCacheEntry(42); + var entry2 = new TestCacheEntry2("test"); + + cache.Set(entry1); + cache.Set(entry2); + + var retrieved1 = cache.Get(); + var retrieved2 = cache.Get(); + + Assert.IsNotNull(retrieved1, "First entry should be retrievable"); + Assert.IsNotNull(retrieved2, "Second entry should be retrievable"); + Assert.AreEqual(42, retrieved1.Value, "First entry value mismatch"); + Assert.AreEqual("test", retrieved2.Text, "Second entry text mismatch"); + } + + [TestMethod] + public void TestOverwriteExistingEntry() + { + var cache = new SerializedCache(); + cache.Set(new TestCacheEntry(42)); + cache.Set(new TestCacheEntry(99)); + + var retrieved = cache.Get(); + Assert.IsNotNull(retrieved, "Entry should exist"); + Assert.AreEqual(99, retrieved.Value, "Entry should be overwritten with new value"); + } + + [TestMethod] + public void TestCopyFromEmptyCache() + { + var source = new SerializedCache(); + var target = new SerializedCache(); + target.Set(new TestCacheEntry(42)); + + target.CopyFrom(source); + + var entry = target.Get(); + Assert.IsNotNull(entry, "Existing entry should remain after copying from empty cache"); + Assert.AreEqual(42, entry.Value, "Existing entry value should be unchanged"); + } + + [TestMethod] + public void TestThreadSafety() + { + var cache = new SerializedCache(); + var tasks = new System.Threading.Tasks.Task[10]; + + for (int i = 0; i < tasks.Length; i++) + { + var value = i; + tasks[i] = System.Threading.Tasks.Task.Run(() => + { + cache.Set(new TestCacheEntry(value)); + cache.Get(); + if (value % 2 == 0) + cache.Remove(typeof(TestCacheEntry)); + }); + } + + System.Threading.Tasks.Task.WaitAll(tasks); + // If we get here without exceptions, the test passes + // The final state is non-deterministic due to race conditions, + // but the operations should be thread-safe + } + + [TestMethod] + public void TestConcurrentCopyFrom() + { + var source = new SerializedCache(); + var target = new SerializedCache(); + source.Set(new TestCacheEntry(42)); + + var tasks = new System.Threading.Tasks.Task[5]; + for (int i = 0; i < tasks.Length; i++) + { + tasks[i] = System.Threading.Tasks.Task.Run(() => + { + target.CopyFrom(source); + }); + } + + System.Threading.Tasks.Task.WaitAll(tasks); + var entry = target.Get(); + Assert.IsNotNull(entry, "Entry should be copied successfully under concurrent operations"); + Assert.AreEqual(42, entry.Value, "Copied entry should have correct value"); + } + } +} From 6d1d8a004b7da6ba0ebd25d22f4b63e97694f817 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sat, 8 Feb 2025 00:46:41 +0100 Subject: [PATCH 20/32] Clean changes --- src/Neo/Persistence/IReadOnlyStore.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Neo/Persistence/IReadOnlyStore.cs b/src/Neo/Persistence/IReadOnlyStore.cs index f46904ead1..79ff9099be 100644 --- a/src/Neo/Persistence/IReadOnlyStore.cs +++ b/src/Neo/Persistence/IReadOnlyStore.cs @@ -22,11 +22,6 @@ namespace Neo.Persistence /// public interface IReadOnlyStore { - /// - /// Serialized cache - /// - SerializedCache SerializedCache { get; } - /// /// Seeks to the entry with the specified key. /// From 6bd4f42b0d01e1ba916450fe5a931988c81296a1 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sat, 8 Feb 2025 00:57:43 +0100 Subject: [PATCH 21/32] Update IReadOnlyStore.cs --- src/Neo/Persistence/IReadOnlyStore.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Neo/Persistence/IReadOnlyStore.cs b/src/Neo/Persistence/IReadOnlyStore.cs index 79ff9099be..f46904ead1 100644 --- a/src/Neo/Persistence/IReadOnlyStore.cs +++ b/src/Neo/Persistence/IReadOnlyStore.cs @@ -22,6 +22,11 @@ namespace Neo.Persistence /// public interface IReadOnlyStore { + /// + /// Serialized cache + /// + SerializedCache SerializedCache { get; } + /// /// Seeks to the entry with the specified key. /// From e4e813069e82f2b2afb4a761ba3f527c53a94af3 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 9 Feb 2025 03:28:46 -0800 Subject: [PATCH 22/32] Update src/Neo/SmartContract/Native/NeoToken.cs Co-authored-by: Christopher Schuchardt --- src/Neo/SmartContract/Native/NeoToken.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Neo/SmartContract/Native/NeoToken.cs b/src/Neo/SmartContract/Native/NeoToken.cs index 6466791c27..d9490c15a3 100644 --- a/src/Neo/SmartContract/Native/NeoToken.cs +++ b/src/Neo/SmartContract/Native/NeoToken.cs @@ -213,7 +213,7 @@ internal override ContractTask InitializeAsync(ApplicationEngine engine, Hardfor var initIndex = engine.PersistingBlock?.Index ?? 0u; var cachedCommittee = new CachedCommittee(engine.ProtocolSettings.StandbyCommittee.Select(p => (p, BigInteger.Zero))); engine.SnapshotCache.Add(CreateStorageKey(Prefix_Committee), new StorageItem(cachedCommittee)); - engine.SnapshotCache.Add(_votersCount, new StorageItem(Array.Empty())); + engine.SnapshotCache.Add(_votersCount, new StorageItem([])); engine.SnapshotCache.Add(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(initIndex), new LastGasPerBlock(5 * GAS.Factor, initIndex)); engine.SnapshotCache.Add(_registerPrice, new StorageItem(1000 * GAS.Factor)); return Mint(engine, Contract.GetBFTAddress(engine.ProtocolSettings.StandbyValidators), TotalAmount, false); From c6a7c000dbea6a9ce16e105268bca645edf8eb7a Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Tue, 11 Feb 2025 11:49:29 +0100 Subject: [PATCH 23/32] Clean Store and Snapshot --- src/Neo/Persistence/MemorySnapshot.cs | 5 +---- src/Neo/Persistence/MemoryStore.cs | 2 +- src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs | 5 +---- src/Plugins/LevelDBStore/Plugins/Storage/Store.cs | 2 +- src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs | 5 +---- src/Plugins/RocksDBStore/Plugins/Storage/Store.cs | 2 +- 6 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/Neo/Persistence/MemorySnapshot.cs b/src/Neo/Persistence/MemorySnapshot.cs index b5ca2fe3d0..2da2ec2849 100644 --- a/src/Neo/Persistence/MemorySnapshot.cs +++ b/src/Neo/Persistence/MemorySnapshot.cs @@ -29,15 +29,12 @@ internal class MemorySnapshot : IStoreSnapshot private readonly ImmutableDictionary _immutableData; private readonly ConcurrentDictionary _writeBatch; - public SerializedCache SerializedCache { get; } - public IStore Store { get; } - internal MemorySnapshot(MemoryStore store, ConcurrentDictionary innerData, SerializedCache serializedCache) + internal MemorySnapshot(MemoryStore store, ConcurrentDictionary innerData) { Store = store; _innerData = innerData; - SerializedCache = serializedCache; _immutableData = innerData.ToImmutableDictionary(ByteArrayEqualityComparer.Default); _writeBatch = new ConcurrentDictionary(ByteArrayEqualityComparer.Default); } diff --git a/src/Neo/Persistence/MemoryStore.cs b/src/Neo/Persistence/MemoryStore.cs index 8db4880c86..ce72979911 100644 --- a/src/Neo/Persistence/MemoryStore.cs +++ b/src/Neo/Persistence/MemoryStore.cs @@ -39,7 +39,7 @@ public void Dispose() { } [MethodImpl(MethodImplOptions.AggressiveInlining)] public IStoreSnapshot GetSnapshot() { - return new MemorySnapshot(this, _innerData, SerializedCache); + return new MemorySnapshot(this, _innerData); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs b/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs index 592a09a1a2..3b73b9a6a7 100644 --- a/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs +++ b/src/Plugins/LevelDBStore/Plugins/Storage/Snapshot.cs @@ -36,16 +36,13 @@ internal class Snapshot : IStoreSnapshot, IEnumerable - new Snapshot(this, _db, SerializedCache); + new Snapshot(this, _db); public void Put(byte[] key, byte[] value) => _db.Put(WriteOptions.Default, key, value); diff --git a/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs b/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs index b708b2b23d..06584bd604 100644 --- a/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs +++ b/src/Plugins/RocksDBStore/Plugins/Storage/Snapshot.cs @@ -28,8 +28,6 @@ internal class Snapshot : IStoreSnapshot private readonly WriteBatch _batch; private readonly ReadOptions _options; - public SerializedCache SerializedCache { get; } - #if NET9_0_OR_GREATER private readonly Lock _lock = new(); #else @@ -38,10 +36,9 @@ internal class Snapshot : IStoreSnapshot public IStore Store { get; } - internal Snapshot(Store store, RocksDb db, SerializedCache serializedCache) + internal Snapshot(Store store, RocksDb db) { Store = store; - SerializedCache = serializedCache; _db = db; _snapshot = db.CreateSnapshot(); _batch = new WriteBatch(); diff --git a/src/Plugins/RocksDBStore/Plugins/Storage/Store.cs b/src/Plugins/RocksDBStore/Plugins/Storage/Store.cs index 2068282a6c..54a9b319fc 100644 --- a/src/Plugins/RocksDBStore/Plugins/Storage/Store.cs +++ b/src/Plugins/RocksDBStore/Plugins/Storage/Store.cs @@ -36,7 +36,7 @@ public void Dispose() public IStoreSnapshot GetSnapshot() { - return new Snapshot(this, _db, SerializedCache); + return new Snapshot(this, _db); } /// From e503094aea3e75703b273d7612a3263531f0e2b4 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Tue, 11 Feb 2025 11:53:28 +0100 Subject: [PATCH 24/32] Clean code --- src/Neo/Persistence/DataCache.cs | 16 ++-------------- src/Neo/Persistence/ICacheableReadOnlyStore.cs | 5 ----- .../Cryptography/MPTTrie/UT_Trie.cs | 13 ++----------- 3 files changed, 4 insertions(+), 30 deletions(-) diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index 2a3a1d8bf3..d547174c08 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -145,21 +145,9 @@ public void Add(StorageKey key, T value) where T : IStorageCacheEntry /// Note: This method does not read the internal storage to check whether the record already exists. public void AddToCache(T? value = default) where T : IStorageCacheEntry { - AddToCache(typeof(T), value); - } - - /// - /// Adds a new entry to the cache. - /// - /// Cache type - /// The data of the entry. - /// The entry has already been cached. - /// Note: This method does not read the internal storage to check whether the record already exists. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void AddToCache(Type type, IStorageCacheEntry? value) - { - _serializedCacheChanges.Set(type, value); + var type = typeof(T); SerializedCache.Remove(type); + _serializedCacheChanges.Set(type, value); } /// diff --git a/src/Neo/Persistence/ICacheableReadOnlyStore.cs b/src/Neo/Persistence/ICacheableReadOnlyStore.cs index c4e584b2f4..8da45120ea 100644 --- a/src/Neo/Persistence/ICacheableReadOnlyStore.cs +++ b/src/Neo/Persistence/ICacheableReadOnlyStore.cs @@ -26,11 +26,6 @@ public interface ICacheableReadOnlyStore : ICacheableReadOnlyStore public interface ICacheableReadOnlyStore : IReadOnlyStore { - /// - /// Serialized cache - /// - public SerializedCache SerializedCache { get; } - /// /// Tries to get the entry from cache. /// diff --git a/tests/Neo.Cryptography.MPTTrie.Tests/Cryptography/MPTTrie/UT_Trie.cs b/tests/Neo.Cryptography.MPTTrie.Tests/Cryptography/MPTTrie/UT_Trie.cs index 62467bed8a..26d8bb810f 100644 --- a/tests/Neo.Cryptography.MPTTrie.Tests/Cryptography/MPTTrie/UT_Trie.cs +++ b/tests/Neo.Cryptography.MPTTrie.Tests/Cryptography/MPTTrie/UT_Trie.cs @@ -25,9 +25,7 @@ namespace Neo.Cryptography.MPTTrie.Tests class TestSnapshot : IStoreSnapshot { public Dictionary store = new(ByteArrayEqualityComparer.Default); - public SerializedCache SerializedCache { get; } = new(); - - private byte[] StoreKey(byte[] key) + private static byte[] StoreKey(byte[] key) { return [.. key]; } @@ -64,13 +62,6 @@ public bool TryGet(byte[] key, [NotNullWhen(true)] out byte[]? value) public void Dispose() { throw new NotImplementedException(); } - public T? GetFromCache() where T : IStorageCacheEntry - { - return default; - } - - public void AddToCache(T? value = default) where T : IStorageCacheEntry { } - public int Size => store.Count; } @@ -80,7 +71,7 @@ public class UT_Trie private readonly Node root; private readonly IStore mptdb; - private void PutToStore(IStore store, Node node) + private static void PutToStore(IStore store, Node node) { store.Put([.. new byte[] { 0xf0 }, .. node.Hash.ToArray()], node.ToArray()); } From 13b3e50c72759d20460ae5bbfd1ff3a987890199 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Tue, 11 Feb 2025 11:55:44 +0100 Subject: [PATCH 25/32] Add ut --- src/Neo/SmartContract/Native/ContractMethodMetadata.cs | 2 +- .../SmartContract/Native/UT_ContractMethodAttribute.cs | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Neo/SmartContract/Native/ContractMethodMetadata.cs b/src/Neo/SmartContract/Native/ContractMethodMetadata.cs index e4d17827a4..c2aaa4e638 100644 --- a/src/Neo/SmartContract/Native/ContractMethodMetadata.cs +++ b/src/Neo/SmartContract/Native/ContractMethodMetadata.cs @@ -54,7 +54,7 @@ public ContractMethodMetadata(MemberInfo member, ContractMethodAttribute attribu if (parameterInfos.Length > 0) { NeedApplicationEngine = parameterInfos[0].ParameterType.IsAssignableFrom(typeof(ApplicationEngine)); - // snapshot is a DataCache instance, and DataCache implements IReadOnlyStoreView + // snapshot is a DataCache instance, and DataCache implements IReadOnlyStore NeedSnapshot = parameterInfos[0].ParameterType.IsAssignableFrom(typeof(DataCache)); } if (NeedApplicationEngine || NeedSnapshot) diff --git a/tests/Neo.UnitTests/SmartContract/Native/UT_ContractMethodAttribute.cs b/tests/Neo.UnitTests/SmartContract/Native/UT_ContractMethodAttribute.cs index 7337507696..34fc2273db 100644 --- a/tests/Neo.UnitTests/SmartContract/Native/UT_ContractMethodAttribute.cs +++ b/tests/Neo.UnitTests/SmartContract/Native/UT_ContractMethodAttribute.cs @@ -37,6 +37,9 @@ class NeedSnapshot [ContractMethod] public bool MethodReadOnlyStoreView(IReadOnlyStore view) => view is null; + [ContractMethod] + public bool MethodCachedReadOnlyStoreView(ICacheableReadOnlyStore view) => view is null; + [ContractMethod] public bool MethodDataCache(DataCache dataCache) => dataCache is null; } From 3bf224080cc6ae1c5e8c4a91e83d73c8525410e0 Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 11 Feb 2025 21:02:32 +0100 Subject: [PATCH 26/32] Update src/Neo/Persistence/SerializedCache.cs Co-authored-by: nan01ab --- src/Neo/Persistence/SerializedCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Neo/Persistence/SerializedCache.cs b/src/Neo/Persistence/SerializedCache.cs index 588bd827b5..4e6b725857 100644 --- a/src/Neo/Persistence/SerializedCache.cs +++ b/src/Neo/Persistence/SerializedCache.cs @@ -25,7 +25,7 @@ public class SerializedCache /// Get cached entry /// /// Type - /// Cache + /// The cached entry. If not cached, the default value will be returned public T? Get() { if (_cache.TryGetValue(typeof(T), out var ret)) From 53215b76e6abb68f7e93b2378cc084ddf3bd7b73 Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 11 Feb 2025 21:03:09 +0100 Subject: [PATCH 27/32] Update src/Neo/Persistence/SerializedCache.cs Co-authored-by: nan01ab --- src/Neo/Persistence/SerializedCache.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Neo/Persistence/SerializedCache.cs b/src/Neo/Persistence/SerializedCache.cs index 4e6b725857..0142fffa3f 100644 --- a/src/Neo/Persistence/SerializedCache.cs +++ b/src/Neo/Persistence/SerializedCache.cs @@ -95,6 +95,7 @@ public void Clear() /// Value public void CopyFrom(SerializedCache value) { + if (ReferenceEquals(this, value)) return; lock (_cache) lock (value._cache) { foreach (var serialized in value._cache) From 64b279648af912e3d60599d23e1e78d3fd1d2479 Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 11 Feb 2025 21:05:51 +0100 Subject: [PATCH 28/32] Update benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs --- benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs b/benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs index a172ee391a..db5fbb6370 100644 --- a/benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs +++ b/benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs @@ -19,7 +19,6 @@ namespace Neo.Benchmark { public class Benchmarks_Cache { - // 256 KiB readonly MemoryStore _store; readonly StoreCache _snapshot; From 1d971bf09a853e3eafc4d55a191e47b14460a5c6 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Thu, 13 Feb 2025 11:49:01 +0100 Subject: [PATCH 29/32] Rename to Upsert --- src/Neo/Persistence/DataCache.cs | 2 +- src/Neo/SmartContract/Native/NeoToken.cs | 2 +- src/Neo/SmartContract/Native/PolicyContract.cs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index a6acabaef8..fc007192d7 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -434,7 +434,7 @@ public bool Contains(StorageKey key) /// /// The key of the entry. /// Serialized cache - public void GetAndChange(StorageKey key, T serializedCache) where T : IStorageCacheEntry + public void Upsert(StorageKey key, T serializedCache) where T : IStorageCacheEntry { var ret = GetAndChange(key, serializedCache.GetStorageItem); ret!.FromReplica(serializedCache.GetStorageItem()); diff --git a/src/Neo/SmartContract/Native/NeoToken.cs b/src/Neo/SmartContract/Native/NeoToken.cs index a5ca66a200..63d8672757 100644 --- a/src/Neo/SmartContract/Native/NeoToken.cs +++ b/src/Neo/SmartContract/Native/NeoToken.cs @@ -294,7 +294,7 @@ private void SetGasPerBlock(ApplicationEngine engine, BigInteger gasPerBlock) if (!CheckCommittee(engine)) throw new InvalidOperationException(); var index = engine.PersistingBlock.Index + 1; - engine.SnapshotCache.GetAndChange(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(index), new LastGasPerBlock(gasPerBlock, index)); + engine.SnapshotCache.Upsert(CreateStorageKey(Prefix_GasPerBlock).AddBigEndian(index), new LastGasPerBlock(gasPerBlock, index)); } /// diff --git a/src/Neo/SmartContract/Native/PolicyContract.cs b/src/Neo/SmartContract/Native/PolicyContract.cs index b79a95e752..eba7a1b102 100644 --- a/src/Neo/SmartContract/Native/PolicyContract.cs +++ b/src/Neo/SmartContract/Native/PolicyContract.cs @@ -201,7 +201,7 @@ private void SetFeePerByte(ApplicationEngine engine, long value) { if (value < 0 || value > 1_00000000) throw new ArgumentOutOfRangeException(nameof(value)); if (!CheckCommittee(engine)) throw new InvalidOperationException(); - engine.SnapshotCache.GetAndChange(_feePerByte, new LastFeePerByte(value)); + engine.SnapshotCache.Upsert(_feePerByte, new LastFeePerByte(value)); } [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.States)] @@ -209,7 +209,7 @@ private void SetExecFeeFactor(ApplicationEngine engine, uint value) { if (value == 0 || value > MaxExecFeeFactor) throw new ArgumentOutOfRangeException(nameof(value)); if (!CheckCommittee(engine)) throw new InvalidOperationException(); - engine.SnapshotCache.GetAndChange(_execFeeFactor, new LastExecFee(value)); + engine.SnapshotCache.Upsert(_execFeeFactor, new LastExecFee(value)); } [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.States)] @@ -217,7 +217,7 @@ private void SetStoragePrice(ApplicationEngine engine, uint value) { if (value == 0 || value > MaxStoragePrice) throw new ArgumentOutOfRangeException(nameof(value)); if (!CheckCommittee(engine)) throw new InvalidOperationException(); - engine.SnapshotCache.GetAndChange(_storagePrice, new LastStorageFee(value)); + engine.SnapshotCache.Upsert(_storagePrice, new LastStorageFee(value)); } [ContractMethod(CpuFee = 1 << 15, RequiredCallFlags = CallFlags.States)] From 894f979f9ff99e9ba43d6336be23eddc8eb25f7e Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Thu, 13 Feb 2025 12:01:06 +0100 Subject: [PATCH 30/32] Fix clone --- src/Neo/Persistence/ClonedCache.cs | 2 +- src/Neo/Persistence/DataCache.cs | 15 ++-- .../Persistence/UT_SerializedCache.cs | 69 +++++++++++++++---- 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/src/Neo/Persistence/ClonedCache.cs b/src/Neo/Persistence/ClonedCache.cs index 5eaa8212fb..b298064a43 100644 --- a/src/Neo/Persistence/ClonedCache.cs +++ b/src/Neo/Persistence/ClonedCache.cs @@ -20,7 +20,7 @@ class ClonedCache : DataCache { private readonly DataCache _innerCache; - public ClonedCache(DataCache innerCache) : base(innerCache.SerializedCache) + public ClonedCache(DataCache innerCache) : base(innerCache.SerializedCacheChanges) { _innerCache = innerCache; } diff --git a/src/Neo/Persistence/DataCache.cs b/src/Neo/Persistence/DataCache.cs index fc007192d7..2c76e7b3f3 100644 --- a/src/Neo/Persistence/DataCache.cs +++ b/src/Neo/Persistence/DataCache.cs @@ -42,7 +42,6 @@ public class Trackable(StorageItem item, TrackState state) public TrackState State { get; set; } = state; } - private readonly SerializedCache _serializedCacheChanges = new(); private readonly Dictionary _dictionary = []; private readonly HashSet _changeSet = []; @@ -51,6 +50,11 @@ public class Trackable(StorageItem item, TrackState state) /// public SerializedCache SerializedCache { get; } + /// + /// This is where the cache changes are stored + /// + internal SerializedCache SerializedCacheChanges { get; } = new(); + /// /// Constructor /// @@ -141,8 +145,7 @@ public void Add(StorageKey key, T value) where T : IStorageCacheEntry public void AddToCache(T? value = default) where T : IStorageCacheEntry { var type = typeof(T); - SerializedCache.Remove(type); - _serializedCacheChanges.Set(type, value); + SerializedCacheChanges.Set(type, value); } /// @@ -152,7 +155,7 @@ public void AddToCache(T? value = default) where T : IStorageCacheEntry /// The entry if found, null otherwise. public T? GetFromCache() where T : IStorageCacheEntry { - var value = _serializedCacheChanges.Get(); + var value = SerializedCacheChanges.Get(); if (value != null) { @@ -195,8 +198,8 @@ public virtual void Commit() break; } } - SerializedCache.CopyFrom(_serializedCacheChanges); - _serializedCacheChanges.Clear(); + SerializedCache.CopyFrom(SerializedCacheChanges); + SerializedCacheChanges.Clear(); _changeSet.Clear(); } } diff --git a/tests/Neo.UnitTests/Persistence/UT_SerializedCache.cs b/tests/Neo.UnitTests/Persistence/UT_SerializedCache.cs index 5fb567ef0c..b65cd5c25a 100644 --- a/tests/Neo.UnitTests/Persistence/UT_SerializedCache.cs +++ b/tests/Neo.UnitTests/Persistence/UT_SerializedCache.cs @@ -11,6 +11,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Neo.Persistence; +using Neo.Persistence.Providers; using Neo.SmartContract; using System; @@ -40,12 +41,52 @@ public void TestGetReturnsDefaultWhenNotSet() Assert.IsNull(cache.Get(), "Expected null when cache does not contain the type"); } + [TestMethod] + public void TestSnapstot() + { + var store = new MemoryStore(); + var entry = new TestCacheEntry(42); + store.SerializedCache.Set(entry); + var retrieved = store.SerializedCache.Get(); + Assert.AreEqual(42, retrieved.Value, "Retrieved entry does not match the set value"); + + var snapshot = new StoreCache(store.GetSnapshot()); + entry = new TestCacheEntry(43); + snapshot.Upsert(new StorageKey(new byte[10]), entry); + + retrieved = store.SerializedCache.Get(); + Assert.AreEqual(42, retrieved.Value, "Retrieved entry does not match the set value"); + retrieved = snapshot.GetFromCache(); + Assert.AreEqual(43, retrieved.Value, "Retrieved entry does not match the set value"); + + var clone = snapshot.CloneCache(); + + entry = new TestCacheEntry(44); + clone.Upsert(new StorageKey(new byte[10]), entry); + retrieved = store.SerializedCache.Get(); + Assert.AreEqual(42, retrieved.Value, "Retrieved entry does not match the set value"); + retrieved = snapshot.GetFromCache(); + Assert.AreEqual(43, retrieved.Value, "Retrieved entry does not match the set value"); + retrieved = clone.GetFromCache(); + Assert.AreEqual(44, retrieved.Value, "Retrieved entry does not match the set value"); + + clone.Commit(); + retrieved = store.SerializedCache.Get(); + Assert.AreEqual(42, retrieved.Value, "Retrieved entry does not match the set value"); + retrieved = snapshot.GetFromCache(); + Assert.AreEqual(44, retrieved.Value, "Retrieved entry does not match the set value"); + + snapshot.Commit(); + retrieved = store.SerializedCache.Get(); + Assert.AreEqual(44, retrieved.Value, "Retrieved entry does not match the set value"); + } + [TestMethod] public void TestSetAndGet() { var entry = new TestCacheEntry(42); var cache = new SerializedCache(); - cache.Set(entry); + cache.Set(entry); var retrieved = cache.Get(); Assert.IsNotNull(retrieved, "Entry was not set correctly"); Assert.AreEqual(42, retrieved.Value, "Retrieved entry does not match the set value"); @@ -56,7 +97,7 @@ public void TestRemove() { var entry = new TestCacheEntry(42); var cache = new SerializedCache(); - cache.Set(entry); + cache.Set(entry); cache.Remove(typeof(TestCacheEntry)); Assert.IsNull(cache.Get(), "Entry should be null after removal"); } @@ -65,8 +106,8 @@ public void TestRemove() public void TestClear() { var cache = new SerializedCache(); - cache.Set(new TestCacheEntry(1)); - cache.Set(new TestCacheEntry2("one")); + cache.Set(new TestCacheEntry(1)); + cache.Set(new TestCacheEntry2("one")); cache.Clear(); Assert.IsNull(cache.Get(), "Cache should be cleared for first type"); Assert.IsNull(cache.Get(), "Cache should be cleared for second type"); @@ -76,10 +117,10 @@ public void TestClear() public void TestCopyFrom() { var source = new SerializedCache(); - source.Set(new TestCacheEntry(99)); + source.Set(new TestCacheEntry(99)); var target = new SerializedCache(); - target.Set(new TestCacheEntry2("hello")); + target.Set(new TestCacheEntry2("hello")); target.CopyFrom(source); @@ -97,7 +138,7 @@ public void TestSetNullValueRemovesEntry() { var cache = new SerializedCache(); var entry = new TestCacheEntry(42); - cache.Set(entry); + cache.Set(entry); Assert.IsNotNull(cache.Get(), "Entry should be set initially"); cache.Set(null); @@ -111,8 +152,8 @@ public void TestMultipleTypesStorage() var entry1 = new TestCacheEntry(42); var entry2 = new TestCacheEntry2("test"); - cache.Set(entry1); - cache.Set(entry2); + cache.Set(entry1); + cache.Set(entry2); var retrieved1 = cache.Get(); var retrieved2 = cache.Get(); @@ -127,8 +168,8 @@ public void TestMultipleTypesStorage() public void TestOverwriteExistingEntry() { var cache = new SerializedCache(); - cache.Set(new TestCacheEntry(42)); - cache.Set(new TestCacheEntry(99)); + cache.Set(new TestCacheEntry(42)); + cache.Set(new TestCacheEntry(99)); var retrieved = cache.Get(); Assert.IsNotNull(retrieved, "Entry should exist"); @@ -140,7 +181,7 @@ public void TestCopyFromEmptyCache() { var source = new SerializedCache(); var target = new SerializedCache(); - target.Set(new TestCacheEntry(42)); + target.Set(new TestCacheEntry(42)); target.CopyFrom(source); @@ -160,7 +201,7 @@ public void TestThreadSafety() var value = i; tasks[i] = System.Threading.Tasks.Task.Run(() => { - cache.Set(new TestCacheEntry(value)); + cache.Set(new TestCacheEntry(value)); cache.Get(); if (value % 2 == 0) cache.Remove(typeof(TestCacheEntry)); @@ -178,7 +219,7 @@ public void TestConcurrentCopyFrom() { var source = new SerializedCache(); var target = new SerializedCache(); - source.Set(new TestCacheEntry(42)); + source.Set(new TestCacheEntry(42)); var tasks = new System.Threading.Tasks.Task[5]; for (int i = 0; i < tasks.Length; i++) From 3a7bf7a0c99db35d17de827ba7b5e29f5e4a5bf8 Mon Sep 17 00:00:00 2001 From: Fernando Diaz Toledano Date: Thu, 13 Feb 2025 12:44:59 +0100 Subject: [PATCH 31/32] Fix --- benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs b/benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs index db5fbb6370..f31cedd079 100644 --- a/benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs +++ b/benchmarks/Neo.Benchmarks/Benchmarks.Cache.cs @@ -11,6 +11,7 @@ using BenchmarkDotNet.Attributes; using Neo.Persistence; +using Neo.Persistence.Providers; using Neo.SmartContract; using Neo.SmartContract.Native; using System.Numerics; From fa7ec14f36f31f58d6f283dd1ee122bb1552ca84 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sat, 15 Feb 2025 17:44:49 +0100 Subject: [PATCH 32/32] Update Program.cs --- benchmarks/Neo.Benchmarks/Program.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/Neo.Benchmarks/Program.cs b/benchmarks/Neo.Benchmarks/Program.cs index fd0e0a2927..420f2781ad 100644 --- a/benchmarks/Neo.Benchmarks/Program.cs +++ b/benchmarks/Neo.Benchmarks/Program.cs @@ -11,6 +11,8 @@ using BenchmarkDotNet.Running; using Neo.Benchmark; +using Neo.Benchmarks.Persistence.Benchmarks; +using Neo.SmartContract.Benchmark; // List all bencharks: // dotnet run -c Release --framework [for example: net9.0] -- --list flat(or tree)