diff --git a/src/Nethermind/Nethermind.Consensus.Test/BlockCachePreWarmerTests.cs b/src/Nethermind/Nethermind.Consensus.Test/BlockCachePreWarmerTests.cs index be2b5306104a..a079f2af6565 100644 --- a/src/Nethermind/Nethermind.Consensus.Test/BlockCachePreWarmerTests.cs +++ b/src/Nethermind/Nethermind.Consensus.Test/BlockCachePreWarmerTests.cs @@ -411,16 +411,16 @@ private sealed class DisposalTrackingPolicy( PreBlockCaches caches, ConcurrentBag created, ConcurrentBag disposed) - : IPooledObjectPolicy + : IPooledObjectPolicy { - public IReadOnlyTxProcessorSource Create() + public IPreBlockCacheWarmupSource Create() { - TrackingEnv env = new(factory.Create(caches), disposed); + TrackingEnv env = new((IPreBlockCacheWarmupSource)factory.Create(caches), disposed); created.Add(env); return env; } - public bool Return(IReadOnlyTxProcessorSource obj) => true; + public bool Return(IPreBlockCacheWarmupSource obj) => true; /// /// Wraps an inner env and records itself in when @@ -428,13 +428,16 @@ public IReadOnlyTxProcessorSource Create() /// disposed by pool eviction from those still retained. /// private sealed class TrackingEnv( - IReadOnlyTxProcessorSource inner, + IPreBlockCacheWarmupSource inner, ConcurrentBag disposed) - : IReadOnlyTxProcessorSource + : IPreBlockCacheWarmupSource { public IReadOnlyTxProcessingScope Build(BlockHeader? baseBlock) => inner.Build(baseBlock); + public IPreBlockCacheWarmupSession BuildPreBlockCacheWarmup(BlockHeader? baseBlock) => + inner.BuildPreBlockCacheWarmup(baseBlock); + public void Dispose() { disposed.Add(this); diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockAccessListManager.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockAccessListManager.cs index ba75b3d56012..4464dbae8a26 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockAccessListManager.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockAccessListManager.cs @@ -905,7 +905,10 @@ private TxProcessorWithWorldState NewProcessor() DefaultObjectPoolProvider provider = new() { MaximumRetained = ProcessorPoolSize }; if (prewarmerEnvFactory is not null && preBlockCaches is not null) { - return provider.Create(new BlockCachePreWarmer.ReadOnlyTxProcessingEnvPooledObjectPolicy(prewarmerEnvFactory, preBlockCaches)); + // The prewarmer's policy hands out the tighter IPreBlockCacheWarmupSource; here we + // only need IReadOnlyTxProcessorSource - upcast at the policy boundary. + return provider.Create(new PrewarmerPolicyAdapter( + new BlockCachePreWarmer.ReadOnlyTxProcessingEnvPooledObjectPolicy(prewarmerEnvFactory, preBlockCaches))); } return readOnlyTxProcessingEnvFactory is not null @@ -913,6 +916,13 @@ private TxProcessorWithWorldState NewProcessor() : null; } + private sealed class PrewarmerPolicyAdapter(BlockCachePreWarmer.ReadOnlyTxProcessingEnvPooledObjectPolicy inner) + : IPooledObjectPolicy + { + public IReadOnlyTxProcessorSource Create() => inner.Create(); + public bool Return(IReadOnlyTxProcessorSource obj) => inner.Return((IPreBlockCacheWarmupSource)obj); + } + private static BlockHeader CreateParentStateHeader(Block block, Hash256 stateRoot) { Hash256 parentHash = block.ParentHash ?? Keccak.Zero; diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs index b99ea1349f2e..fc4cce124ffc 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.ObjectPool; @@ -25,11 +26,16 @@ namespace Nethermind.Consensus.Processing; +internal interface IPreBlockCacheWarmupSource : IReadOnlyTxProcessorSource +{ + IPreBlockCacheWarmupSession BuildPreBlockCacheWarmup(BlockHeader? baseBlock); +} + public sealed class BlockCachePreWarmer : IBlockCachePreWarmer { private readonly int _concurrencyLevel; private readonly bool _parallelExecutionBatchRead; - private readonly ObjectPool _envPool; + private readonly ObjectPool _envPool; private readonly ILogger _logger; private readonly PreBlockCaches _preBlockCaches; private readonly NodeStorageCache _nodeStorageCache; @@ -51,7 +57,7 @@ ILogManager logManager logManager) => _parallelExecutionEnabled = blocksConfig.ParallelExecution; internal BlockCachePreWarmer( - IPooledObjectPolicy poolPolicy, + IPooledObjectPolicy poolPolicy, int maxPoolSize, int concurrency, bool parallelExecutionBatchRead, @@ -154,7 +160,7 @@ private void PreWarmCachesParallel(BlockState blockState, Block suggestedBlock, } } - private void WarmupWithdrawals(ParallelOptions parallelOptions, IReleaseSpec spec, Block block, BlockHeader? parent) + private void WarmupWithdrawals(ParallelOptions parallelOptions, IReleaseSpec spec, Block block, BlockHeader parent) { if (parallelOptions.CancellationToken.IsCancellationRequested) return; @@ -162,25 +168,27 @@ private void WarmupWithdrawals(ParallelOptions parallelOptions, IReleaseSpec spe { if (spec.WithdrawalsEnabled && block.Withdrawals is not null) { - ParallelUnbalancedWork.For(0, block.Withdrawals.Length, parallelOptions, (EnvPool: _envPool, Block: block, Parent: parent), + if (TryWarmupWithdrawalsShared(parallelOptions, block, parent)) + { + return; + } + + PreBlockCacheWarmingState baseState = new(_envPool, block, parent); + + ParallelUnbalancedWork.For(0, block.Withdrawals.Length, parallelOptions, baseState.InitThreadState, static (i, state) => { - IReadOnlyTxProcessorSource env = state.EnvPool.Get(); try { - using IReadOnlyTxProcessingScope scope = env.Build(state.Parent); - scope.WorldState.WarmUp(state.Block.Withdrawals![i].Address); + state.Scope!.WarmUp(state.Payload.Withdrawals![i].Address); } catch (MissingTrieNodeException) { } - finally - { - state.EnvPool.Return(env); - } return state; - }); + }, + PreBlockCacheWarmingState.FinallyAction); } } catch (OperationCanceledException) @@ -193,6 +201,56 @@ private void WarmupWithdrawals(ParallelOptions parallelOptions, IReleaseSpec spe } } + private bool TryWarmupWithdrawalsShared(ParallelOptions parallelOptions, Block block, BlockHeader parent) => + TryWarmupShared(parallelOptions, _envPool, parent, block, block.Withdrawals!.Length, + static (i, payload, warmup) => + { + try + { + warmup.WarmUp(payload.Withdrawals![i].Address); + } + catch (MissingTrieNodeException) + { + } + }); + + /// + /// Acquires an env, builds a shared , and runs the per-item + /// body in parallel - but only when the session reports . + /// Returns false when sharing is unavailable so callers can fall back to per-thread sessions. + /// + private static bool TryWarmupShared( + ParallelOptions parallelOptions, + ObjectPool envPool, + BlockHeader parent, + TPayload payload, + int count, + Action body) + { + IPreBlockCacheWarmupSource env = envPool.Get(); + try + { + using IPreBlockCacheWarmupSession warmup = env.BuildPreBlockCacheWarmup(parent); + if (!warmup.CanBeShared) + { + return false; + } + + ParallelUnbalancedWork.For(0, count, parallelOptions, (payload, warmup, body), + static (i, state) => + { + state.body(i, state.payload, state.warmup); + return state; + }); + + return true; + } + finally + { + envPool.Return(env); + } + } + private void WarmupTransactions(BlockState blockState, ParallelOptions parallelOptions) { if (parallelOptions.CancellationToken.IsCancellationRequested) return; @@ -204,12 +262,12 @@ private void WarmupTransactions(BlockState blockState, ParallelOptions parallelO // Group transactions by sender to process same-sender transactions sequentially // This ensures state changes (balance, storage) from tx[N] are visible to tx[N+1] - Dictionary>? senderGroups = GroupTransactionsBySender(block); + Dictionary? senderGroups = GroupTransactionsBySender(block); try { // Convert to array for parallel iteration - using ArrayPoolList> groupArray = senderGroups.Values.ToPooledList(); + using ArrayPoolList groupArray = senderGroups.Values.ToPooledList(); // Parallel across different senders, sequential within the same sender ParallelUnbalancedWork.For( @@ -219,11 +277,11 @@ private void WarmupTransactions(BlockState blockState, ParallelOptions parallelO (blockState, groupArray, parallelOptions.CancellationToken), static (groupIndex, tupleState) => { - (BlockState? blockState, ArrayPoolList> groups, CancellationToken token) = tupleState; - ArrayPoolList<(int Index, Transaction Tx)>? txList = groups[groupIndex]; + (BlockState? blockState, ArrayPoolList groups, CancellationToken token) = tupleState; + SenderTxGroup txGroup = groups[groupIndex]; // Get thread-local processing state for this sender's transactions - IReadOnlyTxProcessorSource env = blockState.PreWarmer._envPool.Get(); + IPreBlockCacheWarmupSource env = blockState.PreWarmer._envPool.Get(); try { using IReadOnlyTxProcessingScope scope = env.Build(blockState.Parent); @@ -231,9 +289,21 @@ private void WarmupTransactions(BlockState blockState, ParallelOptions parallelO scope.TransactionProcessor.SetBlockExecutionContext(context); // Sequential within the same sender-state changes propagate correctly - foreach ((int txIndex, Transaction? tx) in txList.AsSpan()) + if (token.IsCancellationRequested) return tupleState; + WarmupTransaction(txGroup.FirstTxIndex); + + if (txGroup.RestTxIndexes is not null) + { + foreach (int txIndex in txGroup.RestTxIndexes.AsSpan()) + { + if (token.IsCancellationRequested) return tupleState; + WarmupTransaction(txIndex); + } + } + + void WarmupTransaction(int txIndex) { - if (token.IsCancellationRequested) return tupleState; + Transaction tx = blockState.Block.Transactions[txIndex]; WarmupSingleTransaction(scope, tx, txIndex, blockState); } } @@ -247,7 +317,7 @@ private void WarmupTransactions(BlockState blockState, ParallelOptions parallelO } finally { - foreach (KeyValuePair> kvp in senderGroups) + foreach (KeyValuePair kvp in senderGroups) kvp.Value.Dispose(); } } @@ -261,26 +331,43 @@ private void WarmupTransactions(BlockState blockState, ParallelOptions parallelO } } - private static Dictionary> GroupTransactionsBySender(Block block) + private static Dictionary GroupTransactionsBySender(Block block) { - Dictionary> groups = new(); + Dictionary groups = new(); for (int i = 0; i < block.Transactions.Length; i++) { Transaction tx = block.Transactions[i]; - Address sender = tx.SenderAddress!; + AddressAsKey sender = tx.SenderAddress!; - if (!groups.TryGetValue(sender, out ArrayPoolList<(int, Transaction)> list)) + ref SenderTxGroup group = ref CollectionsMarshal.GetValueRefOrAddDefault(groups, sender, out bool exists); + if (!exists) { - list = new(4); - groups[sender] = list; + group = new(i); + } + else + { + group.Add(i); } - list.Add((i, tx)); } return groups; } + private struct SenderTxGroup(int firstTxIndex) : IDisposable + { + public readonly int FirstTxIndex = firstTxIndex; + public ArrayPoolList? RestTxIndexes; + + public void Add(int txIndex) + { + RestTxIndexes ??= new(4); + RestTxIndexes.Add(txIndex); + } + + public void Dispose() => RestTxIndexes?.Dispose(); + } + private static void WarmupSingleTransaction( IReadOnlyTxProcessingScope scope, Transaction tx, @@ -370,19 +457,19 @@ private void WarmupAddresses(ParallelOptions parallelOptions, Block block) return; } - ObjectPool envPool = PreWarmer._envPool; + ObjectPool envPool = PreWarmer._envPool; try { if (SystemTxAccessLists is not null) { - IReadOnlyTxProcessorSource env = envPool.Get(); + IPreBlockCacheWarmupSource env = envPool.Get(); try { - using IReadOnlyTxProcessingScope scope = env.Build(parent); + using IPreBlockCacheWarmupSession warmup = env.BuildPreBlockCacheWarmup(parent); foreach (AccessList list in SystemTxAccessLists.AsSpan()) { - scope.WorldState.WarmUp(list); + warmup.WarmUp(list); } } finally @@ -398,21 +485,26 @@ private void WarmupAddresses(ParallelOptions parallelOptions, Block block) } else { - WarmingState baseState = new(envPool, block, parent); + if (TryWarmupTransactionAddressesShared(parallelOptions, envPool, block)) + { + return; + } + + PreBlockCacheWarmingState baseState = new(envPool, block, parent); ParallelUnbalancedWork.For( 0, block.Transactions.Length, parallelOptions, baseState.InitThreadState, - static (i, state) => - { - Transaction tx = state.Payload.Transactions[i]; - WarmupSender(tx.SenderAddress, tx.To, state.Scope!.WorldState); + static (i, state) => + { + Transaction tx = state.Payload.Transactions[i]; + WarmupSender(tx.SenderAddress, tx.To, state.Scope!); - return state; - }, - WarmingState.FinallyAction); + return state; + }, + PreBlockCacheWarmingState.FinallyAction); } } catch (OperationCanceledException) @@ -421,11 +513,16 @@ private void WarmupAddresses(ParallelOptions parallelOptions, Block block) } } - private void WarmupFromBal(ParallelOptions parallelOptions, ObjectPool envPool) + private void WarmupFromBal(ParallelOptions parallelOptions, ObjectPool envPool) { using ArrayPoolList accounts = Bal!.AccountChanges.ToPooledList(Bal!.AccountChanges.Count); - WarmingState> baseState = new(envPool, accounts, parent); + if (TryWarmupFromBalShared(parallelOptions, envPool, accounts)) + { + return; + } + + PreBlockCacheWarmingState> baseState = new(envPool, accounts, parent); ParallelUnbalancedWork.For( 0, @@ -435,21 +532,33 @@ private void WarmupFromBal(ParallelOptions parallelOptions, ObjectPool { AccountChanges ac = state.Payload[i]; - IWorldState worldState = state.Scope!.WorldState; + IPreBlockCacheWarmupSession warmup = state.Scope!; - WarmupBalAccount(ac, worldState); + WarmupBalAccount(ac, warmup); return state; }, - WarmingState>.FinallyAction); + PreBlockCacheWarmingState>.FinallyAction); } - private static void WarmupBalAccount(AccountChanges ac, IWorldState worldState) + private bool TryWarmupTransactionAddressesShared(ParallelOptions parallelOptions, ObjectPool envPool, Block block) => + TryWarmupShared(parallelOptions, envPool, parent, block, block.Transactions.Length, + static (i, payload, warmup) => + { + Transaction tx = payload.Transactions[i]; + WarmupSender(tx.SenderAddress, tx.To, warmup); + }); + + private bool TryWarmupFromBalShared(ParallelOptions parallelOptions, ObjectPool envPool, ArrayPoolList accounts) => + TryWarmupShared(parallelOptions, envPool, parent, accounts, accounts.Count, + static (i, payload, warmup) => WarmupBalAccount(payload[i], warmup)); + + private static void WarmupBalAccount(AccountChanges ac, IPreBlockCacheWarmupSession warmup) { try { Address address = ac.Address; - worldState.WarmUp(address); + warmup.WarmUp(address); // Merge two sorted sequences (ChangedSlots, SortedStorageReads) into one // ascending pass for better trie path locality @@ -477,7 +586,7 @@ private static void WarmupBalAccount(AccountChanges ac, IWorldState worldState) readIndex++; } } - worldState.Get(new StorageCell(address, slot)); + warmup.Get(new StorageCell(address, slot)); } } catch (MissingTrieNodeException) @@ -485,18 +594,18 @@ private static void WarmupBalAccount(AccountChanges ac, IWorldState worldState) } } - private static void WarmupSender(Address? sender, Address? to, IWorldState worldState) + private static void WarmupSender(Address? sender, Address? to, IPreBlockCacheWarmupSession warmup) { try { if (sender is not null) { - worldState.WarmUp(sender); + warmup.WarmUp(sender); } if (to is not null) { - worldState.WarmUp(to); + warmup.WarmUp(to); } } catch (MissingTrieNodeException) @@ -505,25 +614,25 @@ private static void WarmupSender(Address? sender, Address? to, IWorldState world } } - private readonly struct WarmingState(ObjectPool envPool, TPayload payload, BlockHeader parent) : IDisposable + private readonly struct PreBlockCacheWarmingState(ObjectPool envPool, TPayload payload, BlockHeader parent) : IDisposable { - public static Action> FinallyAction { get; } = DisposeThreadState; + public static Action> FinallyAction { get; } = DisposeThreadState; - private readonly ObjectPool EnvPool = envPool; - private readonly IReadOnlyTxProcessorSource? Env; + private readonly ObjectPool EnvPool = envPool; + private readonly IPreBlockCacheWarmupSource? Env; public readonly TPayload Payload = payload; - public readonly IReadOnlyTxProcessingScope? Scope; + public readonly IPreBlockCacheWarmupSession? Scope; - private WarmingState(ObjectPool envPool, TPayload payload, BlockHeader parent, IReadOnlyTxProcessorSource env, IReadOnlyTxProcessingScope scope) : this(envPool, payload, parent) + private PreBlockCacheWarmingState(ObjectPool envPool, TPayload payload, BlockHeader parent, IPreBlockCacheWarmupSource env, IPreBlockCacheWarmupSession scope) : this(envPool, payload, parent) { Env = env; Scope = scope; } - public WarmingState InitThreadState() + public PreBlockCacheWarmingState InitThreadState() { - IReadOnlyTxProcessorSource env = EnvPool.Get(); - return new(EnvPool, Payload, parent, env, scope: env.Build(parent)); + IPreBlockCacheWarmupSource env = EnvPool.Get(); + return new(EnvPool, Payload, parent, env, env.BuildPreBlockCacheWarmup(parent)); } public void Dispose() @@ -535,22 +644,25 @@ public void Dispose() } } - private static void DisposeThreadState(WarmingState state) => state.Dispose(); + private static void DisposeThreadState(PreBlockCacheWarmingState state) => state.Dispose(); } /// - /// Pool policy for envs used by the prewarmer. + /// Pool policy for envs used by the prewarmer. /// - internal class ReadOnlyTxProcessingEnvPooledObjectPolicy(PrewarmerEnvFactory envFactory, PreBlockCaches _preBlockCaches) : IPooledObjectPolicy + internal class ReadOnlyTxProcessingEnvPooledObjectPolicy(PrewarmerEnvFactory envFactory, PreBlockCaches _preBlockCaches) : IPooledObjectPolicy { - public IReadOnlyTxProcessorSource Create() => envFactory.Create(_preBlockCaches); + // The factory always builds a PrewarmerTxProcessingEnv, which implements both interfaces. + // The cast is justified by the factory contract; if it ever fails, the prewarmer is + // misconfigured and refusing to start is the right behavior. + public IPreBlockCacheWarmupSource Create() => (IPreBlockCacheWarmupSource)envFactory.Create(_preBlockCaches); /// /// Always returns true — the env is valid for reuse. The pool that owns this policy /// must call on any item it cannot retain; failing /// to do so leaks resources held by the env for the lifetime of the process. /// - public bool Return(IReadOnlyTxProcessorSource obj) => true; + public bool Return(IPreBlockCacheWarmupSource obj) => true; } private record BlockState(BlockCachePreWarmer PreWarmer, Block Block, BlockHeader Parent, IReleaseSpec Spec); diff --git a/src/Nethermind/Nethermind.Consensus/Processing/PrewarmerEnvFactory.cs b/src/Nethermind/Nethermind.Consensus/Processing/PrewarmerEnvFactory.cs index 5270bc36e7c4..aeadf51ae6e6 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/PrewarmerEnvFactory.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/PrewarmerEnvFactory.cs @@ -26,6 +26,21 @@ public IReadOnlyTxProcessorSource Create(PreBlockCaches preBlockCaches) .AddSingleton(); }); - return childScope.Resolve(); + return new PrewarmerTxProcessingEnv( + childScope.Resolve(), + worldState); + } + + private sealed class PrewarmerTxProcessingEnv( + IReadOnlyTxProcessorSource txProcessingEnv, + IPreBlockCacheWarmup preBlockCacheWarmup) + : IReadOnlyTxProcessorSource, IPreBlockCacheWarmupSource + { + public IReadOnlyTxProcessingScope Build(BlockHeader? baseBlock) => txProcessingEnv.Build(baseBlock); + + public IPreBlockCacheWarmupSession BuildPreBlockCacheWarmup(BlockHeader? baseBlock) + => preBlockCacheWarmup.BeginPreBlockCacheWarmup(baseBlock); + + public void Dispose() => txProcessingEnv.Dispose(); } } diff --git a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingTrieStore.cs b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingTrieStore.cs index 767dd9bf41db..449e03815810 100644 --- a/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingTrieStore.cs +++ b/src/Nethermind/Nethermind.Consensus/Stateless/WitnessCapturingTrieStore.cs @@ -16,18 +16,19 @@ namespace Nethermind.Consensus.Stateless; /// Delegates all logic to base store except for writing trie nodes (readonly!) /// Adds logic for capturing trie nodes accessed during execution and state root recomputation. /// -public class WitnessCapturingTrieStore(IReadOnlyTrieStore baseStore) : ITrieStore +public class WitnessCapturingTrieStore(IReadOnlyTrieStore baseStore) : ITrieStore, IScopedReadOnlyTraversalProvider { + private readonly IReadOnlyTrieStore _baseStore = baseStore; private readonly ConcurrentDictionary _rlpCollector = new(); public IEnumerable TouchedNodesRlp => _rlpCollector.Select(static kvp => kvp.Value); - public void Dispose() => baseStore.Dispose(); + public void Dispose() => _baseStore.Dispose(); public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 hash) { - TrieNode node = baseStore.FindCachedOrUnknown(address, in path, hash); - if (node.NodeType != NodeType.Unknown) _rlpCollector.TryAdd(node.Keccak, node.FullRlp.ToArray()); + TrieNode node = _baseStore.FindCachedOrUnknown(address, in path, hash); + CaptureNode(node); return node; } @@ -37,21 +38,59 @@ public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) { - byte[]? rlp = baseStore.TryLoadRlp(address, in path, hash, flags); - if (rlp is not null) _rlpCollector.TryAdd(hash, rlp); + byte[]? rlp = _baseStore.TryLoadRlp(address, in path, hash, flags); + CaptureRlp(hash, rlp); return rlp; } - public bool HasRoot(Hash256 stateRoot) => baseStore.HasRoot(stateRoot); + public bool HasRoot(Hash256 stateRoot) => _baseStore.HasRoot(stateRoot); - public IDisposable BeginScope(BlockHeader? baseBlock) => baseStore.BeginScope(baseBlock); + public IDisposable BeginScope(BlockHeader? baseBlock) => _baseStore.BeginScope(baseBlock); public IScopedTrieStore GetTrieStore(Hash256? address) => new ScopedTrieStore(this, address); - public INodeStorage.KeyScheme Scheme => baseStore.Scheme; + public INodeStorage.KeyScheme Scheme => _baseStore.Scheme; public IBlockCommitter BeginBlockCommit(long blockNumber) => NullCommitter.Instance; // WitnessCapturingTrieStore is read-only, so we return a no-op committer that doesn't persist any trie nodes public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) => NullCommitter.Instance; + + private void CaptureNode(TrieNode node) + { + if (node.NodeType != NodeType.Unknown) + { + _rlpCollector.TryAdd(node.Keccak, node.FullRlp.ToArray()); + } + } + + private void CaptureRlp(Hash256 hash, byte[]? rlp) + { + if (rlp is not null) + { + _rlpCollector.TryAdd(hash, rlp); + } + } + + public ITrieNodeResolver? GetReadOnlyTraversalResolver(Hash256? address) => + _baseStore.GetTrieStore(address) is ITrieNodeResolverSource source + && source.GetReadOnlyTraversalResolver() is { } readOnlyResolver + ? new WitnessCapturingReadOnlyTraversalResolver(this, address, readOnlyResolver) + : null; + + private sealed class WitnessCapturingReadOnlyTraversalResolver( + WitnessCapturingTrieStore fullTrieStore, + Hash256? address, + ITrieNodeResolver inner) : ReadOnlyTraversalResolverBase(fullTrieStore, address) + { + public override TrieNode FindCachedOrUnknown(in TreePath path, Hash256 hash) + { + TrieNode node = inner.FindCachedOrUnknown(path, hash); + fullTrieStore.CaptureNode(node); + return node; + } + + protected override ITrieNodeResolver WithAddress(Hash256? address1) => + new WitnessCapturingReadOnlyTraversalResolver(fullTrieStore, address1, inner.GetStorageTrieNodeResolver(address1)); + } } diff --git a/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs b/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs index 42dd8800e659..36dc274370b7 100644 --- a/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs +++ b/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs @@ -10,10 +10,9 @@ namespace Nethermind.Core.Buffers; /// -/// Basically like ArraySegment, but only contain length, which reduces it size from 16byte to 12byte. Useful for -/// polling memory where memory pool usually can't return exactly the same size of data. To conserve space, The -/// underlying array can be null and this struct is meant to be non nullable, checking the `IsNull` property to check -/// if it represent null. +/// Like ArraySegment but with explicit offset+length, supporting zero-copy logical slices over a shared +/// backing array. Useful for pooled memory where the pool may return a larger buffer than requested. The +/// underlying array can be null; check the property to detect that case. /// public readonly struct CappedArray where T : struct { @@ -25,11 +24,18 @@ namespace Nethermind.Core.Buffers; public static object EmptyBoxed { get; } = _empty; private readonly T[]? _array; + private readonly int _offset; private readonly int _length; public CappedArray(T[]? array, int length) + : this(array, 0, length) + { + } + + public CappedArray(T[]? array, int offset, int length) { _array = array; + _offset = offset; _length = length; } @@ -38,8 +44,15 @@ public CappedArray(T[]? array) if (array is not null) { _array = array; + _offset = 0; _length = array.Length; } + else + { + _array = null; + _offset = 0; + _length = 0; + } } public static implicit operator ReadOnlySpan(in CappedArray array) => array.AsSpan(); @@ -50,23 +63,39 @@ public T this[int index] { get { + // Validate against the logical slice first - a negative `index` would otherwise wrap into + // `_offset + index` and silently read bytes from before the slice (e.g. parent RLP for an + // inline trie child). The underlying-array check defends against an oversized _length passed + // to the (array, offset, length) constructor. + if ((uint)index >= (uint)_length) + { + ThrowArgumentOutOfRangeException(); + } + T[] array = _array!; - if (index >= _length || (uint)index >= (uint)array.Length) + int arrayIndex = _offset + index; + if ((uint)arrayIndex >= (uint)array.Length) { ThrowArgumentOutOfRangeException(); } - return array[index]; + return array[arrayIndex]; } set { + if ((uint)index >= (uint)_length) + { + ThrowArgumentOutOfRangeException(); + } + T[] array = _array!; - if (index >= _length || (uint)index >= (uint)array.Length) + int arrayIndex = _offset + index; + if ((uint)arrayIndex >= (uint)array.Length) { ThrowArgumentOutOfRangeException(); } - array[index] = value; + array[arrayIndex] = value; } } @@ -74,15 +103,26 @@ public T this[int index] private static void ThrowArgumentOutOfRangeException() => throw new ArgumentOutOfRangeException(); public int Length => _length; + public int Offset => _offset; public int UnderlyingLength => _array?.Length ?? 0; public T[]? UnderlyingArray => _array; - public bool IsUncapped => _length == _array?.Length; + public bool IsUncapped => _offset == 0 && _length == _array?.Length; public bool IsNull => _array is null; public bool IsNotNull => _array is not null; public bool IsNullOrEmpty => _length == 0; public bool IsNotNullOrEmpty => _length > 0; - public Span AsSpan() => _array.AsSpan(0, _length); - public Span AsSpan(int start, int length) => _array.AsSpan(start, length); + public Span AsSpan() => _array.AsSpan(_offset, _length); + public Span AsSpan(int start, int length) + { + // Validate against the logical slice first; otherwise a negative `start` would wrap into + // `_offset + start` and yield a span over bytes outside the slice (e.g. parent RLP). + if ((uint)start > (uint)_length || (uint)length > (uint)(_length - start)) + { + ThrowArgumentOutOfRangeException(); + } + + return _array.AsSpan(_offset + start, length); + } public T[]? ToArray() { @@ -90,7 +130,7 @@ public T this[int index] if (array is null) return null; if (array.Length == 0) return []; - if (_length == array.Length) return array; + if (_offset == 0 && _length == array.Length) return array; return AsSpan().ToArray(); } @@ -99,5 +139,14 @@ public T this[int index] : base.ToString(); public ArraySegment AsArraySegment() => AsArraySegment(0, _length); - public ArraySegment AsArraySegment(int start, int length) => new(_array!, start, length); + public ArraySegment AsArraySegment(int start, int length) + { + // See AsSpan(start, length) - validate against the logical slice before adding _offset. + if ((uint)start > (uint)_length || (uint)length > (uint)(_length - start)) + { + ThrowArgumentOutOfRangeException(); + } + + return new(_array!, _offset + start, length); + } } diff --git a/src/Nethermind/Nethermind.Evm/State/IPreBlockCacheWarmup.cs b/src/Nethermind/Nethermind.Evm/State/IPreBlockCacheWarmup.cs new file mode 100644 index 000000000000..f9b0ffa56c2c --- /dev/null +++ b/src/Nethermind/Nethermind.Evm/State/IPreBlockCacheWarmup.cs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core; +using Nethermind.Core.Eip2930; + +namespace Nethermind.Evm.State; + +/// +/// Creates direct pre-block cache warmup sessions for account and storage reads. +/// +public interface IPreBlockCacheWarmup +{ + /// + /// Begins a direct account and storage cache warmup session rooted at . + /// + IPreBlockCacheWarmupSession BeginPreBlockCacheWarmup(BlockHeader? baseBlock); +} + +/// +/// Direct account and storage reader used only for pre-block cache warmup. +/// +public interface IPreBlockCacheWarmupSession : IDisposable +{ + /// + /// Whether this session can be used concurrently by multiple warmup workers. + /// + bool CanBeShared { get; } + + /// + /// Warms the account cache entry for . + /// + bool WarmUp(Address address); + + /// + /// Warms and returns the storage cache entry for . + /// + ReadOnlySpan Get(in StorageCell storageCell); +} + +public static class PreBlockCacheWarmupSessionExtensions +{ + /// + /// Walks , warming each address and its storage slots through + /// . Storage slot warmup is skipped when the account doesn't exist + /// (saves the per-slot trie walk for missing accounts). + /// + public static void WarmUp(this IPreBlockCacheWarmupSession session, AccessList accessList) + { + foreach ((Address address, AccessList.StorageKeysEnumerable storages) in accessList) + { + if (!session.WarmUp(address)) continue; + + foreach (Int256.UInt256 storage in storages) + { + session.Get(new StorageCell(address, in storage)); + } + } + } +} diff --git a/src/Nethermind/Nethermind.State.Test/ScopeProviderTests.cs b/src/Nethermind/Nethermind.State.Test/ScopeProviderTests.cs index 3ef890d40613..ff730d6ec046 100644 --- a/src/Nethermind/Nethermind.State.Test/ScopeProviderTests.cs +++ b/src/Nethermind/Nethermind.State.Test/ScopeProviderTests.cs @@ -9,6 +9,7 @@ using Nethermind.Core.Test; using Nethermind.Core.Test.Builders; using Nethermind.Evm.State; +using Nethermind.Int256; using Nethermind.Logging; using Nethermind.State; using NUnit.Framework; @@ -148,4 +149,131 @@ public void Test_NullAccountWithNonEmptyStorageDoesNotThrow() writeBatch.Set(TestItem.AddressA, null); } } + + [TestCase(true, true, 0, 1, 0, 1)] + [TestCase(true, false, 1, 0, 1, 0)] + [TestCase(false, true, 2, 0, 1, 0)] + public void PrewarmerScope_UsesUncachedReadersOnlyInsideExplicitWarmup( + bool populatePreBlockCache, + bool beginPreBlockCacheWarmup, + int expectedGetCount, + int expectedUncachedGetCount, + int expectedCreateStorageTreeCount, + int expectedUncachedCreateStorageTreeCount) + { + PreBlockCaches preBlockCaches = new(); + CountingScopeProvider baseProvider = new(); + PrewarmerScopeProvider provider = new(baseProvider, preBlockCaches, populatePreBlockCache); + + if (beginPreBlockCacheWarmup) + { + using IPreBlockCacheWarmupSession warmup = ((IPreBlockCacheWarmup)provider).BeginPreBlockCacheWarmup(null); + warmup.WarmUp(TestItem.AddressA).Should().BeTrue(); + warmup.WarmUp(TestItem.AddressA).Should().BeTrue(); + warmup.Get(new StorageCell(TestItem.AddressA, 1)).Length.Should().Be(0); + } + else + { + using IWorldStateScopeProvider.IScope scope = provider.BeginScope(null); + scope.Get(TestItem.AddressA).Should().Be(baseProvider.Scope.Account); + scope.Get(TestItem.AddressA).Should().Be(baseProvider.Scope.Account); + scope.CreateStorageTree(TestItem.AddressA).RootHash.Should().Be(baseProvider.Scope.StorageTree.RootHash); + } + + baseProvider.Scope.GetCount.Should().Be(expectedGetCount); + baseProvider.Scope.UncachedGetCount.Should().Be(expectedUncachedGetCount); + baseProvider.Scope.CreateStorageTreeCount.Should().Be(expectedCreateStorageTreeCount); + baseProvider.Scope.UncachedCreateStorageTreeCount.Should().Be(expectedUncachedCreateStorageTreeCount); + } + + private sealed class CountingScopeProvider : IWorldStateScopeProvider + { + public CountingScope Scope { get; } = new(); + + public bool HasRoot(BlockHeader baseBlock) => true; + + public IWorldStateScopeProvider.IScope BeginScope(BlockHeader baseBlock) => Scope; + } + + private sealed class CountingScope : IWorldStateScopeProvider.IScope, IUncachedAccountReader, IUncachedStorageTreeProvider + { + public Account Account { get; } = new(1); + public IWorldStateScopeProvider.IStorageTree StorageTree { get; } = new CountingStorageTree(); + public int GetCount { get; private set; } + public int UncachedGetCount { get; private set; } + public int CreateStorageTreeCount { get; private set; } + public int UncachedCreateStorageTreeCount { get; private set; } + + public void Dispose() + { + } + + public Hash256 RootHash => Keccak.EmptyTreeHash; + + public void UpdateRootHash() + { + } + + public Account Get(Address address) + { + GetCount++; + return Account; + } + + public bool CanReadAccountUncached => true; + + public Account GetAccountUncached(Address address) + { + UncachedGetCount++; + return Account; + } + + public void HintGet(Address address, Account account) + { + } + + public IWorldStateScopeProvider.ICodeDb CodeDb => NullCodeDb.Instance; + + public IWorldStateScopeProvider.IStorageTree CreateStorageTree(Address address) + { + CreateStorageTreeCount++; + return StorageTree; + } + + public bool CanCreateStorageTreeUncachedAccount => true; + + public IWorldStateScopeProvider.IStorageTree CreateStorageTreeUncachedAccount(Address address) + { + UncachedCreateStorageTreeCount++; + return StorageTree; + } + + public IWorldStateScopeProvider.IWorldStateWriteBatch StartWriteBatch(int estimatedAccountNum) => throw new NotImplementedException(); + + public void Commit(long blockNumber) + { + } + } + + private sealed class NullCodeDb : IWorldStateScopeProvider.ICodeDb + { + public static NullCodeDb Instance { get; } = new(); + + public byte[] GetCode(in ValueHash256 codeHash) => []; + + public IWorldStateScopeProvider.ICodeSetter BeginCodeWrite() => throw new NotImplementedException(); + } + + private sealed class CountingStorageTree : IWorldStateScopeProvider.IStorageTree + { + public Hash256 RootHash => Keccak.EmptyTreeHash; + + public byte[] Get(in UInt256 index) => []; + + public void HintSet(in UInt256 index, byte[] value) + { + } + + public byte[] Get(in ValueHash256 hash) => []; + } } diff --git a/src/Nethermind/Nethermind.State/IUncachedAccountReader.cs b/src/Nethermind/Nethermind.State/IUncachedAccountReader.cs new file mode 100644 index 000000000000..188a65292081 --- /dev/null +++ b/src/Nethermind/Nethermind.State/IUncachedAccountReader.cs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Evm.State; + +namespace Nethermind.State; + +internal interface IUncachedAccountReader +{ + bool CanReadAccountUncached { get; } + + Account? GetAccountUncached(Address address); +} + +internal interface IUncachedStorageTreeProvider +{ + bool CanCreateStorageTreeUncachedAccount { get; } + + IWorldStateScopeProvider.IStorageTree CreateStorageTreeUncachedAccount(Address address); +} diff --git a/src/Nethermind/Nethermind.State/PrewarmerScopeProvider.cs b/src/Nethermind/Nethermind.State/PrewarmerScopeProvider.cs index 52ba7ec4e865..fb8789a59aa2 100644 --- a/src/Nethermind/Nethermind.State/PrewarmerScopeProvider.cs +++ b/src/Nethermind/Nethermind.State/PrewarmerScopeProvider.cs @@ -32,26 +32,61 @@ public class PrewarmerScopeProvider( IWorldStateScopeProvider baseProvider, PreBlockCaches preBlockCaches, bool populatePreBlockCache = true -) : IWorldStateScopeProvider, IPreBlockCaches +) : IWorldStateScopeProvider, IPreBlockCaches, IPreBlockCacheWarmup { public bool HasRoot(BlockHeader? baseBlock) => baseProvider.HasRoot(baseBlock); - public IWorldStateScopeProvider.IScope BeginScope(BlockHeader? baseBlock) => new ScopeWrapper(baseProvider.BeginScope(baseBlock), preBlockCaches, populatePreBlockCache); + public IWorldStateScopeProvider.IScope BeginScope(BlockHeader? baseBlock) + => new ScopeWrapper(baseProvider.BeginScope(baseBlock), preBlockCaches, populatePreBlockCache, useUncachedReads: false); public PreBlockCaches? Caches => preBlockCaches; public bool IsWarmWorldState => !populatePreBlockCache; - private sealed class ScopeWrapper(IWorldStateScopeProvider.IScope baseScope, PreBlockCaches preBlockCaches, bool populatePreBlockCache) : IWorldStateScopeProvider.IScope + /// + public IPreBlockCacheWarmupSession BeginPreBlockCacheWarmup(BlockHeader? baseBlock) { - private readonly IWorldStateScopeProvider.IScope baseScope = baseScope; - private readonly SeqlockCache preBlockCache = preBlockCaches.StateCache; - private readonly SeqlockCache storageCache = preBlockCaches.StorageCache; - private readonly bool populatePreBlockCache = populatePreBlockCache; + IWorldStateScopeProvider.IScope baseScope = baseProvider.BeginScope(baseBlock); + return new ScopeWrapper(baseScope, preBlockCaches, populatePreBlockCache, populatePreBlockCache && CanUseUncachedReads(baseScope)); + } + + private static bool CanUseUncachedReads(IWorldStateScopeProvider.IScope baseScope) + => baseScope is IUncachedAccountReader { CanReadAccountUncached: true } + && baseScope is IUncachedStorageTreeProvider { CanCreateStorageTreeUncachedAccount: true }; + + private sealed class ScopeWrapper : IWorldStateScopeProvider.IScope, IPreBlockCacheWarmupSession + { + private readonly IWorldStateScopeProvider.IScope baseScope; + private readonly SeqlockCache preBlockCache; + private readonly SeqlockCache storageCache; + private readonly bool populatePreBlockCache; private readonly IMetricObserver _metricObserver = Metrics.PrewarmerGetTime; private readonly bool _measureMetric = Metrics.DetailedMetricsEnabled; - private readonly PrewarmerGetTimeLabels _labels = populatePreBlockCache ? PrewarmerGetTimeLabels.Prewarmer : PrewarmerGetTimeLabels.NonPrewarmer; + private readonly PrewarmerGetTimeLabels _labels; + // When useUncachedReads is true, CanUseUncachedReads has already proven both capability + // interfaces are present and enabled - cache the typed references so the hot read path + // never re-runs the interface type-test. + private readonly IUncachedAccountReader? _uncachedAccountReader; + private readonly IUncachedStorageTreeProvider? _uncachedStorageTreeProvider; private long _writeBatchTime = 0; + public ScopeWrapper( + IWorldStateScopeProvider.IScope baseScope, + PreBlockCaches preBlockCaches, + bool populatePreBlockCache, + bool useUncachedReads) + { + this.baseScope = baseScope; + preBlockCache = preBlockCaches.StateCache; + storageCache = preBlockCaches.StorageCache; + this.populatePreBlockCache = populatePreBlockCache; + _labels = populatePreBlockCache ? PrewarmerGetTimeLabels.Prewarmer : PrewarmerGetTimeLabels.NonPrewarmer; + if (useUncachedReads) + { + _uncachedAccountReader = (IUncachedAccountReader)baseScope; + _uncachedStorageTreeProvider = (IUncachedStorageTreeProvider)baseScope; + } + } + public void Dispose() { if (_measureMetric && _writeBatchTime != 0) @@ -64,11 +99,16 @@ public void Dispose() public IWorldStateScopeProvider.ICodeDb CodeDb => baseScope.CodeDb; public IWorldStateScopeProvider.IStorageTree CreateStorageTree(Address address) => new StorageTreeWrapper( - baseScope.CreateStorageTree(address), + CreateBaseStorageTree(address), storageCache, address, populatePreBlockCache); + private IWorldStateScopeProvider.IStorageTree CreateBaseStorageTree(Address address) => + _uncachedStorageTreeProvider is { } provider + ? provider.CreateStorageTreeUncachedAccount(address) + : baseScope.CreateStorageTree(address); + public IWorldStateScopeProvider.IWorldStateWriteBatch StartWriteBatch(int estimatedAccountNum) { if (!_measureMetric) @@ -151,7 +191,22 @@ public void UpdateRootHash() public void HintGet(Address address, Account? account) => baseScope.HintGet(address, account); - private Account? GetFromBaseTree(in AddressAsKey address) => baseScope.Get(address); + public bool CanBeShared => _uncachedAccountReader is not null; + + public bool WarmUp(Address address) => Get(address) is not null; + + public ReadOnlySpan Get(in StorageCell storageCell) + { + IWorldStateScopeProvider.IStorageTree storageTree = CreateStorageTree(storageCell.Address); + return !storageCell.IsHash + ? storageTree.Get(storageCell.Index) + : storageTree.Get(storageCell.Hash); + } + + private Account? GetFromBaseTree(in AddressAsKey address) => + _uncachedAccountReader is { } reader + ? reader.GetAccountUncached(address) + : baseScope.Get(address); } private sealed class StorageTreeWrapper( @@ -173,6 +228,11 @@ private sealed class StorageTreeWrapper( public byte[] Get(in UInt256 index) { StorageCell storageCell = new(address, in index); // TODO: Make the dictionary use UInt256 directly + return Get(in storageCell); + } + + private byte[] Get(in StorageCell storageCell) + { long sw = _measureMetric ? Stopwatch.GetTimestamp() : 0; if (populatePreBlockCache) { @@ -216,9 +276,7 @@ private byte[] LoadFromTreeStorage(in StorageCell storageCell) : baseStorageTree.Get(storageCell.Hash); } - public byte[] Get(in ValueHash256 hash) => - // Not a critical path. so we just forward for simplicity - baseStorageTree.Get(in hash); + public byte[] Get(in ValueHash256 hash) => Get(new StorageCell(address, hash)); } private class WriteBatchLifetimeMeasurer(IWorldStateScopeProvider.IWorldStateWriteBatch baseWriteBatch, IMetricObserver metricObserver, long startTime, bool populatePreBlockCache) : IWorldStateScopeProvider.IWorldStateWriteBatch diff --git a/src/Nethermind/Nethermind.State/TrieStoreScopeProvider.cs b/src/Nethermind/Nethermind.State/TrieStoreScopeProvider.cs index 870330fb9bf3..188a75aa5885 100644 --- a/src/Nethermind/Nethermind.State/TrieStoreScopeProvider.cs +++ b/src/Nethermind/Nethermind.State/TrieStoreScopeProvider.cs @@ -42,7 +42,7 @@ public IWorldStateScopeProvider.IScope BeginScope(BlockHeader? baseBlock) protected virtual StorageTree CreateStorageTree(Address address, Hash256 storageRoot) => new(_trieStore.GetTrieStore(address), storageRoot, _logManager); - private class TrieStoreWorldStateBackendScope(StateTree backingStateTree, TrieStoreScopeProvider scopeProvider, IWorldStateScopeProvider.ICodeDb codeDb, IDisposable trieStoreCloser, ILogManager logManager) : IWorldStateScopeProvider.IScope + private class TrieStoreWorldStateBackendScope(StateTree backingStateTree, TrieStoreScopeProvider scopeProvider, IWorldStateScopeProvider.ICodeDb codeDb, IDisposable trieStoreCloser, ILogManager logManager) : IWorldStateScopeProvider.IScope, IUncachedAccountReader, IUncachedStorageTreeProvider { public void Dispose() { @@ -65,6 +65,10 @@ public void Dispose() return account; } + public Account? GetAccountUncached(Address address) => _backingStateTree.Get(address); + + public bool CanReadAccountUncached => true; + public void HintGet(Address address, Account? account) => _loadedAccounts.TryAdd(address, account); public IWorldStateScopeProvider.ICodeDb CodeDb => _codeDb1; @@ -108,14 +112,15 @@ public void Commit(long blockNumber) _storages.Clear(); } - internal StorageTree LookupStorageTree(Address address) + internal StorageTree LookupStorageTree(Address address, bool cacheAccount = true) { if (_storages.TryGetValue(address, out StorageTree storageTree)) { return storageTree; } - storageTree = _scopeProvider.CreateStorageTree(address, Get(address)?.StorageRoot ?? Keccak.EmptyTreeHash); + Account? account = cacheAccount ? Get(address) : GetAccountUncached(address); + storageTree = _scopeProvider.CreateStorageTree(address, account?.StorageRoot ?? Keccak.EmptyTreeHash); _storages[address] = storageTree; return storageTree; } @@ -123,6 +128,14 @@ internal StorageTree LookupStorageTree(Address address) public void ClearLoadedAccounts() => _loadedAccounts.Clear(); public IWorldStateScopeProvider.IStorageTree CreateStorageTree(Address address) => LookupStorageTree(address); + + public IWorldStateScopeProvider.IStorageTree CreateStorageTreeUncachedAccount(Address address) + { + Account? account = GetAccountUncached(address); + return _scopeProvider.CreateStorageTree(address, account?.StorageRoot ?? Keccak.EmptyTreeHash); + } + + public bool CanCreateStorageTreeUncachedAccount => true; } private class WorldStateWriteBatch( diff --git a/src/Nethermind/Nethermind.State/WorldStateMetricsScopeProvider.cs b/src/Nethermind/Nethermind.State/WorldStateMetricsScopeProvider.cs index 07a533e8fb8a..77fa1ba64e45 100644 --- a/src/Nethermind/Nethermind.State/WorldStateMetricsScopeProvider.cs +++ b/src/Nethermind/Nethermind.State/WorldStateMetricsScopeProvider.cs @@ -18,34 +18,65 @@ public class WorldStateMetricsScopeProvider(IWorldStateScopeProvider baseProvide public bool HasRoot(BlockHeader? baseBlock) => _baseProvider.HasRoot(baseBlock); public IWorldStateScopeProvider.IScope BeginScope(BlockHeader? baseBlock) => new MetricsScope(_baseProvider.BeginScope(baseBlock), this); - private sealed class MetricsScope(IWorldStateScopeProvider.IScope baseScope, WorldStateMetricsScopeProvider parent) : IWorldStateScopeProvider.IScope + private sealed class MetricsScope : IWorldStateScopeProvider.IScope, IUncachedAccountReader, IUncachedStorageTreeProvider { + private readonly IWorldStateScopeProvider.IScope _baseScope; + private readonly WorldStateMetricsScopeProvider _parent; + // Capability references resolved once; the per-call cost was a virtual property fetch + interface + // type-test on every account/storage read through the decorator. + private readonly IUncachedAccountReader? _uncachedAccountReader; + private readonly IUncachedStorageTreeProvider? _uncachedStorageTreeProvider; + + public MetricsScope(IWorldStateScopeProvider.IScope baseScope, WorldStateMetricsScopeProvider parent) + { + _baseScope = baseScope; + _parent = parent; + if (baseScope is IUncachedAccountReader { CanReadAccountUncached: true } uncachedReader) + { + _uncachedAccountReader = uncachedReader; + } + if (baseScope is IUncachedStorageTreeProvider { CanCreateStorageTreeUncachedAccount: true } uncachedStorage) + { + _uncachedStorageTreeProvider = uncachedStorage; + } + } + public void Dispose() { - baseScope.Dispose(); - parent._stateMerkleizationTime = 0d; + _baseScope.Dispose(); + _parent._stateMerkleizationTime = 0d; } - public Hash256 RootHash => baseScope.RootHash; + public Hash256 RootHash => _baseScope.RootHash; + + public void UpdateRootHash() => _baseScope.UpdateRootHash(); + + public Account? Get(Address address) => _baseScope.Get(address); + + public bool CanReadAccountUncached => _uncachedAccountReader is not null; + + public Account? GetAccountUncached(Address address) => + _uncachedAccountReader is { } reader ? reader.GetAccountUncached(address) : _baseScope.Get(address); - public void UpdateRootHash() => baseScope.UpdateRootHash(); + public void HintGet(Address address, Account? account) => _baseScope.HintGet(address, account); - public Account? Get(Address address) => baseScope.Get(address); + public IWorldStateScopeProvider.ICodeDb CodeDb => _baseScope.CodeDb; - public void HintGet(Address address, Account? account) => baseScope.HintGet(address, account); + public IWorldStateScopeProvider.IStorageTree CreateStorageTree(Address address) => _baseScope.CreateStorageTree(address); - public IWorldStateScopeProvider.ICodeDb CodeDb => baseScope.CodeDb; + public bool CanCreateStorageTreeUncachedAccount => _uncachedStorageTreeProvider is not null; - public IWorldStateScopeProvider.IStorageTree CreateStorageTree(Address address) => baseScope.CreateStorageTree(address); + public IWorldStateScopeProvider.IStorageTree CreateStorageTreeUncachedAccount(Address address) => + _uncachedStorageTreeProvider is { } provider ? provider.CreateStorageTreeUncachedAccount(address) : _baseScope.CreateStorageTree(address); - public IWorldStateScopeProvider.IWorldStateWriteBatch StartWriteBatch(int estimatedAccountNum) => baseScope.StartWriteBatch(estimatedAccountNum); + public IWorldStateScopeProvider.IWorldStateWriteBatch StartWriteBatch(int estimatedAccountNum) => _baseScope.StartWriteBatch(estimatedAccountNum); public void Commit(long blockNumber) { long start = Stopwatch.GetTimestamp(); - baseScope.Commit(blockNumber); - parent._stateMerkleizationTime += Stopwatch.GetElapsedTime(start).TotalMilliseconds; - parent._updateMetrics(parent._stateMerkleizationTime); + _baseScope.Commit(blockNumber); + _parent._stateMerkleizationTime += Stopwatch.GetElapsedTime(start).TotalMilliseconds; + _parent._updateMetrics(_parent._stateMerkleizationTime); } } } diff --git a/src/Nethermind/Nethermind.State/WorldStateScopeOperationLogger.cs b/src/Nethermind/Nethermind.State/WorldStateScopeOperationLogger.cs index a02e2c7edece..38fe1b520410 100644 --- a/src/Nethermind/Nethermind.State/WorldStateScopeOperationLogger.cs +++ b/src/Nethermind/Nethermind.State/WorldStateScopeOperationLogger.cs @@ -26,40 +26,82 @@ public IWorldStateScopeProvider.IScope BeginScope(BlockHeader? baseBlock) return new ScopeWrapper(baseScopeProvider.BeginScope(baseBlock), scopeId, _logger); } - private class ScopeWrapper(IWorldStateScopeProvider.IScope innerScope, long scopeId, ILogger logger) : IWorldStateScopeProvider.IScope + private class ScopeWrapper : IWorldStateScopeProvider.IScope, IUncachedAccountReader, IUncachedStorageTreeProvider { + private readonly IWorldStateScopeProvider.IScope _innerScope; + private readonly long _scopeId; + private readonly ILogger _logger; + // See WorldStateMetricsScopeProvider.MetricsScope - capability references resolved once. + private readonly IUncachedAccountReader? _uncachedAccountReader; + private readonly IUncachedStorageTreeProvider? _uncachedStorageTreeProvider; + + public ScopeWrapper(IWorldStateScopeProvider.IScope innerScope, long scopeId, ILogger logger) + { + _innerScope = innerScope; + _scopeId = scopeId; + _logger = logger; + if (innerScope is IUncachedAccountReader { CanReadAccountUncached: true } uncachedReader) + { + _uncachedAccountReader = uncachedReader; + } + if (innerScope is IUncachedStorageTreeProvider { CanCreateStorageTreeUncachedAccount: true } uncachedStorage) + { + _uncachedStorageTreeProvider = uncachedStorage; + } + } + public void Dispose() { - innerScope.Dispose(); - logger.Trace($"{scopeId}: Scope disposed"); + _innerScope.Dispose(); + _logger.Trace($"{_scopeId}: Scope disposed"); } - public Hash256 RootHash => innerScope.RootHash; + public Hash256 RootHash => _innerScope.RootHash; public void UpdateRootHash() { - innerScope.UpdateRootHash(); - logger.Trace($"{scopeId}: Update root hash"); + _innerScope.UpdateRootHash(); + _logger.Trace($"{_scopeId}: Update root hash"); } public Account? Get(Address address) { - Account? res = innerScope.Get(address); - logger.Trace($"{scopeId}: Get account {address}, got {res}"); + Account? res = _innerScope.Get(address); + _logger.Trace($"{_scopeId}: Get account {address}, got {res}"); return res; } - public void HintGet(Address address, Account? account) => innerScope.HintGet(address, account); + public bool CanReadAccountUncached => _uncachedAccountReader is not null; - public IWorldStateScopeProvider.ICodeDb CodeDb => innerScope.CodeDb; + public Account? GetAccountUncached(Address address) + { + Account? res = _uncachedAccountReader is { } reader ? reader.GetAccountUncached(address) : _innerScope.Get(address); + _logger.Trace($"{_scopeId}: Get uncached account {address}, got {res}"); + return res; + } + + public void HintGet(Address address, Account? account) => _innerScope.HintGet(address, account); + + public IWorldStateScopeProvider.ICodeDb CodeDb => _innerScope.CodeDb; public IWorldStateScopeProvider.IStorageTree CreateStorageTree(Address address) => - new StorageTreeWrapper(innerScope.CreateStorageTree(address), address, scopeId, logger); + new StorageTreeWrapper(_innerScope.CreateStorageTree(address), address, _scopeId, _logger); + + public bool CanCreateStorageTreeUncachedAccount => _uncachedStorageTreeProvider is not null; + + public IWorldStateScopeProvider.IStorageTree CreateStorageTreeUncachedAccount(Address address) + { + IWorldStateScopeProvider.IStorageTree storageTree = _uncachedStorageTreeProvider is { } provider + ? provider.CreateStorageTreeUncachedAccount(address) + : _innerScope.CreateStorageTree(address); + _logger.Trace($"{_scopeId}: Create uncached storage tree {address}"); + return new StorageTreeWrapper(storageTree, address, _scopeId, _logger); + } public IWorldStateScopeProvider.IWorldStateWriteBatch StartWriteBatch(int estimatedAccountNum) => - new WriteBatchWrapper(innerScope.StartWriteBatch(estimatedAccountNum), scopeId, logger); + new WriteBatchWrapper(_innerScope.StartWriteBatch(estimatedAccountNum), _scopeId, _logger); - public void Commit(long blockNumber) => innerScope.Commit(blockNumber); + public void Commit(long blockNumber) => _innerScope.Commit(blockNumber); } private class StorageTreeWrapper(IWorldStateScopeProvider.IStorageTree storageTree, Address address, long scopeId, ILogger logger) : IWorldStateScopeProvider.IStorageTree diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapUpperBoundAdapter.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapUpperBoundAdapter.cs index 89f50a307d75..ff8c4c4faae7 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapUpperBoundAdapter.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapUpperBoundAdapter.cs @@ -14,7 +14,7 @@ namespace Nethermind.Synchronization.SnapSync; /// UpperBound. This is to prevent double writes on partitioned snap ranges. /// /// -public class SnapUpperBoundAdapter(IScopedTrieStore baseTrieStore) : IScopedTrieStore +public class SnapUpperBoundAdapter(IScopedTrieStore baseTrieStore) : IScopedTrieStore, ITrieNodeResolverSource { public ValueHash256 UpperBound = ValueKeccak.MaxValue; @@ -30,6 +30,9 @@ public class SnapUpperBoundAdapter(IScopedTrieStore baseTrieStore) : IScopedTrie public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => new BoundedSnapCommitter(baseTrieStore.BeginCommit(root, writeFlags), UpperBound); + public ITrieNodeResolver? GetReadOnlyTraversalResolver() => + baseTrieStore is ITrieNodeResolverSource source ? source.GetReadOnlyTraversalResolver() : null; + private sealed class BoundedSnapCommitter(ICommitter baseCommitter, ValueHash256 subtreeLimit) : ICommitter { public void Dispose() => baseCommitter.Dispose(); diff --git a/src/Nethermind/Nethermind.Trie.Test/OverlayTrieStoreTests.cs b/src/Nethermind/Nethermind.Trie.Test/OverlayTrieStoreTests.cs index 88cc07ab93e1..0c2fea329ce2 100644 --- a/src/Nethermind/Nethermind.Trie.Test/OverlayTrieStoreTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/OverlayTrieStoreTests.cs @@ -75,4 +75,35 @@ public void TrieStore_OverlayExistingStore() // After all this, the original should not change. dbProvider.StateDb.GetAllKeys().Count().Should().Be(originalKeyCount); } + + [Test] + [NonParallelizable] + public void OverlayStore_read_only_lookup_uses_shared_base_nodes() + { + using TrieStore existingStore = TestTrieStoreFactory.Build(new MemDb(), No.Pruning, No.Persistence, LimboLogs.Instance); + PatriciaTree patriciaTree = new(existingStore, LimboLogs.Instance); + { + using IBlockCommitter _ = existingStore.BeginBlockCommit(0); + patriciaTree.Set(TestItem.Keccaks[0].Bytes, TestItem.Keccaks[0].BytesToArray()); + patriciaTree.Set(TestItem.Keccaks[1].Bytes, TestItem.Keccaks[1].BytesToArray()); + patriciaTree.Commit(); + } + + IDbProvider dbProvider = TestMemDbProvider.Init(); + ReadOnlyDbProvider readOnlyDbProvider = dbProvider.AsReadOnly(true); + ITrieStore overlayStore = new OverlayTrieStore(readOnlyDbProvider.GetDb(DbNames.State), existingStore.AsReadOnly()); + + long sharedHitsBefore = existingStore.SharedNodeHitCount; + long clonesBefore = existingStore.CloneForReadOnlyCount; + + PatriciaTree overlaidTree = new(overlayStore, LimboLogs.Instance) + { + RootHash = patriciaTree.RootHash + }; + + overlaidTree.Get(TestItem.Keccaks[0].Bytes).ToArray().Should().BeEquivalentTo(TestItem.Keccaks[0].BytesToArray()); + + existingStore.SharedNodeHitCount.Should().BeGreaterThan(sharedHitsBefore); + existingStore.CloneForReadOnlyCount.Should().Be(clonesBefore); + } } diff --git a/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs b/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs index 33d6d71d408d..c22e7d9f4432 100644 --- a/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs @@ -83,7 +83,7 @@ public void Memory_with_one_node_is_288() { TrieNode trieNode = new(NodeType.Leaf, Keccak.Zero); // 56B - using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new TestPruningStrategy(true)); + using TrieStore fullTrieStore = CreateTrieStore(); TreePath emptyPath = TreePath.Empty; using (ICommitter? committer = fullTrieStore.BeginStateBlockCommit(1234, null)) { @@ -201,7 +201,7 @@ public void Memory_with_two_nodes_is_correct() TrieNode trieNode1 = new(NodeType.Leaf, TestItem.KeccakA); TrieNode trieNode2 = new(NodeType.Leaf, TestItem.KeccakB); - using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new TestPruningStrategy(true)); + using TrieStore fullTrieStore = CreateTrieStore(); TreePath emptyPath = TreePath.Empty; using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(1234, null)) { @@ -775,9 +775,6 @@ public async Task Read_only_trie_store_is_allowing_many_thread_to_work_with_the_ { trieNode.SetChild(i, new TrieNode(NodeType.Unknown, TestItem.Keccaks[i])); } - - trieNode.Seal(); - MemDb memDb = new(); using TrieStore fullTrieStore = CreateTrieStore( kvStore: memDb, @@ -791,7 +788,6 @@ public async Task Read_only_trie_store_is_allowing_many_thread_to_work_with_the_ { committer.CommitNode(ref emptyPath, trieNode); } - using (fullTrieStore.PrepareStableState(default)) { } if (beThreadSafe) { @@ -806,9 +802,9 @@ void CheckChildren() { trieStore.FindCachedOrUnknown(TreePath.Empty, trieNode.Keccak).GetChildHash(i % 16).Should().BeEquivalentTo(TestItem.Keccaks[i % 16], i.ToString()); } - catch (Exception) + catch (Exception exception) { - throw new AssertionException("Failed"); + throw new AssertionException($"Failed: {exception}"); } } } @@ -823,7 +819,7 @@ void CheckChildren() if (beThreadSafe) { - await Task.WhenAll(); + await Task.WhenAll(tasks); } else { @@ -874,9 +870,200 @@ public void ReadOnly_store_returns_copies(bool pruning) readOnlyNode.FullRlp[0].Should().Be(firstReadOnlyByte); } + public enum ReadOnlyVariant { Direct, PreCached, Cached } + + private TrieNode BuildAndCommitSealedBranch(TrieStore fullTrieStore) + { + TrieNode node = new(NodeType.Branch); + for (int i = 0; i < 16; i++) + { + node.SetChild(i, new TrieNode(NodeType.Unknown, TestItem.Keccaks[i])); + } + + IScopedTrieStore scoped = fullTrieStore.GetTrieStore(null); + TreePath emptyPath = TreePath.Empty; + node.ResolveKey(scoped, ref emptyPath); + node.Seal(); + + using (ICommitter committer = fullTrieStore.BeginStateBlockCommit(0, node)) + { + committer.CommitNode(ref emptyPath, node); + } + + return node; + } + + private static void AssertReadServedFromSharedCache( + TrieStore fullTrieStore, + IScopedTrieStore readOnlyScopedStore, + TrieNode node) + { + long sharedHitsBefore = fullTrieStore.SharedNodeHitCount; + long clonesBefore = fullTrieStore.CloneForReadOnlyCount; + long loadedFromCacheBefore = Nethermind.Trie.Pruning.Metrics.LoadedFromCacheNodesCount; + + PatriciaTree readOnlyTree = new(readOnlyScopedStore, LimboLogs.Instance) + { + RootHash = node.Keccak + }; + + readOnlyTree.GetNodeByPath([], node.Keccak)!.Should().Equal(node.FullRlp.AsSpan().ToArray()); + + fullTrieStore.SharedNodeHitCount.Should().BeGreaterThan(sharedHitsBefore); + fullTrieStore.CloneForReadOnlyCount.Should().Be(clonesBefore); + // Regression guard: previous metric under-reporting caused this counter to stay flat + // even when the read was served from cache. Keep this assertion in the shared helper + // so all variants (Direct/PreCached/Cached/CommitBuffer) cover it. + Nethermind.Trie.Pruning.Metrics.LoadedFromCacheNodesCount.Should().BeGreaterThan(loadedFromCacheBefore); + } + + [Test] + [NonParallelizable] + [TestCase(ReadOnlyVariant.Direct)] + [TestCase(ReadOnlyVariant.PreCached)] + [TestCase(ReadOnlyVariant.Cached)] + public void Read_only_lookup_uses_shared_cached_nodes(ReadOnlyVariant variant) + { + using TrieStore fullTrieStore = CreateTrieStore(); + TrieNode node = BuildAndCommitSealedBranch(fullTrieStore); + + IDisposable? extra = null; + IScopedTrieStore scoped; + switch (variant) + { + case ReadOnlyVariant.Direct: + scoped = fullTrieStore.AsReadOnly().GetTrieStore(null); + break; + case ReadOnlyVariant.PreCached: + PreCachedTrieStore pre = new(fullTrieStore.AsReadOnly(), new NodeStorageCache { Enabled = true }); + extra = pre; + scoped = pre.GetTrieStore(null); + break; + case ReadOnlyVariant.Cached: + scoped = new CachedTrieStore(fullTrieStore.AsReadOnly().GetTrieStore(null)); + break; + default: + throw new ArgumentOutOfRangeException(nameof(variant)); + } + + try + { + AssertReadServedFromSharedCache(fullTrieStore, scoped, node); + } + finally + { + extra?.Dispose(); + } + } + + [Test] + [NonParallelizable] + public void Shared_resolver_returns_sealed_cached_node_with_resolvable_children() + { + using TrieStore fullTrieStore = CreateTrieStore(); + TrieNode node = BuildAndCommitSealedBranch(fullTrieStore); + + IScopedTrieStore readOnlyScopedTrieStore = fullTrieStore.AsReadOnly().GetTrieStore(null); + ITrieNodeResolver sharedResolver = ((ITrieNodeResolverSource)readOnlyScopedTrieStore).GetReadOnlyTraversalResolver()!; + TrieNode sharedNode = sharedResolver.FindCachedOrUnknown(TreePath.Empty, node.Keccak); + + sharedNode.IsSealed.Should().BeTrue(); + sharedNode.HasRlp.Should().BeTrue(); + sharedNode.FullRlp.AsSpan().ToArray().Should().Equal(node.FullRlp.AsSpan().ToArray()); + + TreePath childPath = TreePath.Empty; + sharedNode.GetChild(sharedResolver, ref childPath, 0)!.Keccak.Should().Be(TestItem.Keccaks[0]); + childPath = TreePath.Empty; + sharedNode.GetChild(sharedResolver, ref childPath, 0)!.Keccak.Should().Be(TestItem.Keccaks[0]); + } + + [Test] + [NonParallelizable] + public void Shared_read_only_lookup_does_not_clone_cached_ref_only_nodes() + { + using TrieStore fullTrieStore = CreateTrieStore(); + TreePath emptyPath = TreePath.Empty; + fullTrieStore.FindCachedOrUnknown(null, emptyPath, TestItem.KeccakA); + + long fallbacksBefore = fullTrieStore.FallbackNotShareableCount; + long clonesBefore = fullTrieStore.CloneForReadOnlyCount; + + IScopedTrieStore readOnlyScopedTrieStore = fullTrieStore.AsReadOnly().GetTrieStore(null); + ITrieNodeResolver sharedResolver = ((ITrieNodeResolverSource)readOnlyScopedTrieStore).GetReadOnlyTraversalResolver()!; + + TrieNode node = sharedResolver.FindCachedOrUnknown(emptyPath, TestItem.KeccakA); + + node.NodeType.Should().Be(NodeType.Unknown); + node.Keccak.Should().Be(TestItem.KeccakA); + fullTrieStore.FallbackNotShareableCount.Should().BeGreaterThan(fallbacksBefore); + fullTrieStore.CloneForReadOnlyCount.Should().Be(clonesBefore); + } + + [Test] + [NonParallelizable] + public async Task PatriciaTree_read_only_lookup_uses_shared_cached_nodes_in_commit_buffer() + { + ManualResetEvent writeBlocker = new(false); + ManualResetEventSlim writeReached = new(false); + TestMemDb memDb = new(); + memDb.WriteFunc = (_, _) => + { + writeReached.Set(); + writeBlocker.WaitOne(); + return true; + }; + + TestPruningStrategy pruningStrategy = new(shouldPrune: false, deleteObsoleteKeys: true); + using TrieStore fullTrieStore = CreateTrieStore( + kvStore: memDb, + pruningStrategy: pruningStrategy, + persistenceStrategy: No.Persistence, + pruningConfig: new PruningConfig() + { + PruningBoundary = 1, + DirtyNodeShardBit = 4, + MaxBufferedCommitCount = 20, + TrackPastKeys = true + }); + + TrieNode node = BuildAndCommitSealedBranch(fullTrieStore); + + Task pruneTask = Task.Run(() => + { + pruningStrategy.ShouldPruneEnabled = true; + fullTrieStore.SyncPruneQueue(); + pruningStrategy.ShouldPruneEnabled = false; + }); + + writeReached.Wait(1000).Should().BeTrue("the pruning task must hold the trie store in commit-buffer mode"); + + try + { + using (fullTrieStore.BeginScope(Build.A.BlockHeader.WithStateRoot(node.Keccak).TestObject)) + { + fullTrieStore.IsInCommitBufferMode.Should().BeTrue(); + + long clonesBeforeHasRoot = fullTrieStore.CloneForReadOnlyCount; + fullTrieStore.HasRoot(node.Keccak!).Should().BeTrue(); + fullTrieStore.CloneForReadOnlyCount.Should().Be(clonesBeforeHasRoot); + + AssertReadServedFromSharedCache( + fullTrieStore, + fullTrieStore.AsReadOnly().GetTrieStore(null), + node); + } + } + finally + { + writeBlocker.Set(); + await pruneTask; + } + } + private long ExpectedPerNodeKeyMemorySize => (scheme == INodeStorage.KeyScheme.Hash ? 0 : TrieStoreDirtyNodesCache.Key.MemoryUsage) + MemorySizes.ObjectHeaderMethodTable + MemorySizes.RefSize + 4 + MemorySizes.RefSize; [Test] + [NonParallelizable] public void After_commit_should_have_has_root() { MemDb db = new(); @@ -890,7 +1077,9 @@ public void After_commit_should_have_has_root() stateTree.Set(TestItem.AddressA, account); stateTree.Commit(); } + long clonesBefore = trieStore.CloneForReadOnlyCount; trieStore.HasRoot(stateTree.RootHash).Should().BeTrue(); + trieStore.CloneForReadOnlyCount.Should().Be(clonesBefore); stateTree.Get(TestItem.AddressA); account = account.WithChangedBalance(2); @@ -900,7 +1089,9 @@ public void After_commit_should_have_has_root() stateTree.Set(TestItem.AddressA, account); stateTree.Commit(); } + clonesBefore = trieStore.CloneForReadOnlyCount; trieStore.HasRoot(stateTree.RootHash).Should().BeTrue(); + trieStore.CloneForReadOnlyCount.Should().Be(clonesBefore); } [Test] diff --git a/src/Nethermind/Nethermind.Trie.Test/TrieNodeResolverWithReadFlagsTests.cs b/src/Nethermind/Nethermind.Trie.Test/TrieNodeResolverWithReadFlagsTests.cs index 6d28229901d0..295913da3663 100644 --- a/src/Nethermind/Nethermind.Trie.Test/TrieNodeResolverWithReadFlagsTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/TrieNodeResolverWithReadFlagsTests.cs @@ -59,4 +59,56 @@ public void LoadRlp_shouldPassTheFlag_forStorageStoreAlso() memDb.KeyWasReadWithFlags(NodeStorage.GetHalfPathNodeStoragePath(TestItem.KeccakA, TreePath.Empty, theKeccak), theFlags); } + + [Test] + public void ReadOnlyTraversal_shouldPreserveReadFlags() + { + TestResolver sourceResolver = new(); + TrieNodeResolverWithReadFlags resolver = new(sourceResolver, ReadFlags.HintCacheMiss); + ITrieNodeResolver readOnlyResolver = ((ITrieNodeResolverSource)resolver).GetReadOnlyTraversalResolver()!; + + readOnlyResolver.LoadRlp(TreePath.Empty, TestItem.KeccakA, ReadFlags.HintReadAhead); + + Assert.That(sourceResolver.ReadOnlyResolver.LastFlags, Is.EqualTo(ReadFlags.HintCacheMiss | ReadFlags.HintReadAhead)); + } + + private sealed class TestResolver : ITrieNodeResolver, ITrieNodeResolverSource + { + public readonly TestReadOnlyResolver ReadOnlyResolver = new(); + + public TrieNode FindCachedOrUnknown(in TreePath path, Hash256 hash) => new(NodeType.Unknown, hash); + + public byte[]? LoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => []; + + public byte[]? TryLoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => []; + + public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address) => this; + + public INodeStorage.KeyScheme Scheme => INodeStorage.KeyScheme.HalfPath; + + public ITrieNodeResolver? GetReadOnlyTraversalResolver() => ReadOnlyResolver; + } + + private sealed class TestReadOnlyResolver : ITrieNodeResolver + { + public ReadFlags LastFlags { get; private set; } + + public TrieNode FindCachedOrUnknown(in TreePath path, Hash256 hash) => new(NodeType.Unknown, hash); + + public byte[]? LoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) + { + LastFlags = flags; + return []; + } + + public byte[]? TryLoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) + { + LastFlags = flags; + return []; + } + + public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address) => this; + + public INodeStorage.KeyScheme Scheme => INodeStorage.KeyScheme.HalfPath; + } } diff --git a/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs b/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs index 2584978cd7e4..fb34765aef51 100644 --- a/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs @@ -297,6 +297,30 @@ public void Get_child_hash_works_on_inlined_child_of_a_branch() Assert.That(getResult, Is.Null); } + [Test] + public void Inline_child_slice_shares_parent_rlp_array() + { + Context ctx = new(); + TrieNode trieNode = new(NodeType.Branch); + + trieNode[11] = ctx.TiniestLeaf; + TreePath emptyPath = TreePath.Empty; + CappedArray rlp = trieNode.RlpEncode(NullTrieNodeResolver.Instance, ref emptyPath); + TrieNode decoded = new(NodeType.Branch, rlp); + + TrieNode? child = decoded.GetChild(NullTrieNodeResolver.Instance, ref emptyPath, 11); + + child.Should().NotBeNull(); + CappedArray parentRlp = decoded.FullRlp; + CappedArray childRlp = child!.FullRlp; + childRlp.UnderlyingArray.Should().BeSameAs(parentRlp.UnderlyingArray); + childRlp.Offset.Should().BeGreaterThan(parentRlp.Offset); + childRlp.AsSpan().ToArray().Should().Equal(ctx.TiniestLeaf.FullRlp.AsSpan().ToArray()); + + TrieNode clone = child.Clone(); + clone.FullRlp.AsSpan().ToArray().Should().Equal(childRlp.AsSpan().ToArray()); + } + [Test] public void Get_child_hash_works_on_hashed_child_of_an_extension() { @@ -1164,6 +1188,21 @@ public void FullRlp_seqlock_returns_consistent_length_and_array() result.UnderlyingArray.Should().BeSameAs(small); } + [Test] + public void WriteRlp_preserves_non_zero_offset() + { + byte[] backing = [0, 1, 2, 3, 4]; + CappedArray slice = new(backing, 2, 2); + TrieNode node = new(NodeType.Leaf, CappedArray.Empty); + + node.WriteRlp(slice); + + CappedArray result = node.FullRlp; + result.UnderlyingArray.Should().BeSameAs(backing); + result.Offset.Should().Be(2); + result.AsSpan().ToArray().Should().Equal(new byte[] { 2, 3 }); + } + [Test] [Category("LongRunning")] public void FullRlp_concurrent_writers_do_not_corrupt_seqlock() diff --git a/src/Nethermind/Nethermind.Trie/CachedTrieStore.cs b/src/Nethermind/Nethermind.Trie/CachedTrieStore.cs index 93518dfb4ce0..e95e511a2712 100644 --- a/src/Nethermind/Nethermind.Trie/CachedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/CachedTrieStore.cs @@ -15,7 +15,7 @@ namespace Nethermind.Trie; /// will need to seek back. /// /// -public class CachedTrieStore(IScopedTrieStore @base) : IScopedTrieStore +public class CachedTrieStore(IScopedTrieStore @base) : IScopedTrieStore, ITrieNodeResolverSource { private readonly NonBlocking.ConcurrentDictionary<(TreePath path, Hash256 hash), TrieNode> _cachedNode = new(); @@ -35,5 +35,29 @@ public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address) => public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => @base.BeginCommit(root, writeFlags); -} + public ITrieNodeResolver? GetReadOnlyTraversalResolver() => + @base is ITrieNodeResolverSource source + && source.GetReadOnlyTraversalResolver() is ITrieNodeResolver readOnlyResolver + ? new CachedTrieNodeResolver(readOnlyResolver) + : null; + + private sealed class CachedTrieNodeResolver(ITrieNodeResolver inner) : ITrieNodeResolver + { + private readonly NonBlocking.ConcurrentDictionary<(TreePath path, Hash256 hash), TrieNode> _cachedNode = new(); + + public TrieNode FindCachedOrUnknown(in TreePath path, Hash256 hash) => + _cachedNode.GetOrAdd((path, hash), key => inner.FindCachedOrUnknown(key.path, key.hash)); + + public byte[]? LoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => + inner.LoadRlp(in path, hash, flags); + + public byte[]? TryLoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => + inner.TryLoadRlp(in path, hash, flags); + + public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address) => + new CachedTrieNodeResolver(inner.GetStorageTrieNodeResolver(address)); + + public INodeStorage.KeyScheme Scheme => inner.Scheme; + } +} diff --git a/src/Nethermind/Nethermind.Trie/PatriciaTree.cs b/src/Nethermind/Nethermind.Trie/PatriciaTree.cs index d44fbb6bd3f4..b1d4d27b1a29 100644 --- a/src/Nethermind/Nethermind.Trie/PatriciaTree.cs +++ b/src/Nethermind/Nethermind.Trie/PatriciaTree.cs @@ -47,6 +47,7 @@ private static TraverseStack GetTraverseStack() private static void ReturnTraverseStack(TraverseStack stack) => _threadStaticTraverseStack = stack; public readonly IScopedTrieStore TrieStore; + private readonly ITrieNodeResolver _readResolver; public ICappedArrayPool? _bufferPool; private readonly bool _allowCommits; @@ -122,6 +123,7 @@ public PatriciaTree( { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); TrieStore = trieStore ?? throw new ArgumentNullException(nameof(trieStore)); + _readResolver = TrieStore.AsReadOnlyTraversal(); _allowCommits = allowCommits; RootHash = rootHash; @@ -340,7 +342,7 @@ public void SetRootHash(Hash256? value, bool resetObjects) } else if (resetObjects) { - RootRef = TrieStore.FindCachedOrUnknown(TreePath.Empty, _rootHash); + RootRef = _readResolver.FindCachedOrUnknown(TreePath.Empty, _rootHash); } } @@ -364,7 +366,7 @@ public virtual ReadOnlySpan Get(ReadOnlySpan rawKey, Hash256? rootHa if (rootHash is not null) { - root = TrieStore.FindCachedOrUnknown(emptyPath, rootHash); + root = _readResolver.FindCachedOrUnknown(emptyPath, rootHash); } CappedArray result = GetNew(nibbles, ref emptyPath, root, isNodeRead: false); @@ -422,7 +424,7 @@ public void WarmUpPath(ReadOnlySpan rawKey) TrieNode root = RootRef; if (rootHash is not null) { - root = TrieStore.FindCachedOrUnknown(emptyPath, rootHash); + root = _readResolver.FindCachedOrUnknown(emptyPath, rootHash); } CappedArray result = GetNew(nibbles, ref emptyPath, root, isNodeRead: true); return result.ToArray(); @@ -452,7 +454,7 @@ public void WarmUpPath(ReadOnlySpan rawKey) TrieNode root = RootRef; if (rootHash is not null) { - root = TrieStore.FindCachedOrUnknown(emptyPath, rootHash); + root = _readResolver.FindCachedOrUnknown(emptyPath, rootHash); } CappedArray result = GetNew(nibbles, ref emptyPath, root, isNodeRead: true); @@ -927,7 +929,7 @@ private CappedArray GetNew(Span remainingKey, ref TreePath path, Tri return default; } - node.ResolveNode(TrieStore, path); + node.ResolveNode(_readResolver, path); if (isNodeRead && remainingKey.Length == 0) { @@ -936,8 +938,9 @@ private CappedArray GetNew(Span remainingKey, ref TreePath path, Tri if (node.IsLeaf || node.IsExtension) { - int commonPrefixLength = remainingKey.CommonPrefixLength(node.Key); - if (commonPrefixLength == node.Key!.Length) + byte[] key = node.Key!; + int commonPrefixLength = remainingKey.CommonPrefixLength(key); + if (commonPrefixLength == key.Length) { if (node.IsLeaf) { @@ -948,10 +951,9 @@ private CappedArray GetNew(Span remainingKey, ref TreePath path, Tri } // Continue traversal to the child of the extension - path.AppendMut(node.Key); - TrieNode? extensionChild = node.GetChildWithChildPath(TrieStore, ref path, 0); - remainingKey = remainingKey[node!.Key.Length..]; - node = extensionChild; + path.AppendMut(key); + node = node.GetChildWithChildPath(_readResolver, ref path, 0); + remainingKey = remainingKey[key.Length..]; continue; } @@ -962,10 +964,7 @@ private CappedArray GetNew(Span remainingKey, ref TreePath path, Tri int nib = remainingKey[0]; path.AppendMut(nib); - TrieNode? child = node.GetChildWithChildPath(TrieStore, ref path, nib); - - // Continue loop with child as current node - node = child; + node = node.GetChildWithChildPath(_readResolver, ref path, nib); remainingKey = remainingKey[1..]; } } @@ -990,13 +989,18 @@ private void DoWarmUpPath(Span remainingKey, ref TreePath path, TrieNode? } // Call FindCachedOrUnknown on some path. - if (node.IsSealed && node.Keccak is not null && path.Length % 2 == 1) node = TrieStore.FindCachedOrUnknown(path, node!.Keccak); - node.ResolveNode(TrieStore, path); + if (node.IsSealed && node.Keccak is not null && path.Length % 2 == 1) + { + node = _readResolver.FindCachedOrUnknown(path, node.Keccak); + } + + node.ResolveNode(_readResolver, path); if (node.IsLeaf || node.IsExtension) { - int commonPrefixLength = remainingKey.CommonPrefixLength(node.Key); - if (commonPrefixLength == node.Key!.Length) + byte[] key = node.Key!; + int commonPrefixLength = remainingKey.CommonPrefixLength(key); + if (commonPrefixLength == key.Length) { if (node.IsLeaf) { @@ -1005,10 +1009,9 @@ private void DoWarmUpPath(Span remainingKey, ref TreePath path, TrieNode? } // Continue traversal to the child of the extension - path.AppendMut(node.Key); - TrieNode? extensionChild = node.GetChildWithChildPath(TrieStore, ref path, 0, keepChildRef: true); - remainingKey = remainingKey[node!.Key.Length..]; - node = extensionChild; + path.AppendMut(key); + node = node.GetChildWithChildPath(_readResolver, ref path, 0, keepChildRef: true); + remainingKey = remainingKey[key.Length..]; continue; } @@ -1020,10 +1023,7 @@ private void DoWarmUpPath(Span remainingKey, ref TreePath path, TrieNode? int nextNib = remainingKey[0]; path.AppendMut(nextNib); - TrieNode? child = node.GetChildWithChildPath(TrieStore, ref path, nextNib, keepChildRef: true); - - // Continue loop with child as current node - node = child; + node = node.GetChildWithChildPath(_readResolver, ref path, nextNib, keepChildRef: true); remainingKey = remainingKey[1..]; } } diff --git a/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs b/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs index bdf77c4e0719..a7f743e9967e 100644 --- a/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs @@ -12,7 +12,7 @@ namespace Nethermind.Trie; -public sealed class PreCachedTrieStore : ITrieStore +public sealed class PreCachedTrieStore : ITrieStore, IScopedReadOnlyTraversalProvider { private readonly ITrieStore _inner; private readonly NodeStorageCache _preBlockCache; @@ -56,6 +56,24 @@ public PreCachedTrieStore(ITrieStore inner, NodeStorageCache cache) (in key) => _inner.TryLoadRlp(key.Address, in key.Path, key.Hash, flags)); public INodeStorage.KeyScheme Scheme => _inner.Scheme; + + public ITrieNodeResolver? GetReadOnlyTraversalResolver(Hash256? address) => + _inner.GetTrieStore(address) is ITrieNodeResolverSource source + && source.GetReadOnlyTraversalResolver() is { } readOnlyResolver + ? new PreCachedReadOnlyTraversalResolver(this, address, readOnlyResolver) + : null; + + private sealed class PreCachedReadOnlyTraversalResolver( + PreCachedTrieStore fullTrieStore, + Hash256? address, + ITrieNodeResolver inner) : ReadOnlyTraversalResolverBase(fullTrieStore, address) + { + public override TrieNode FindCachedOrUnknown(in TreePath path, Hash256 hash) => + inner.FindCachedOrUnknown(path, hash); + + protected override ITrieNodeResolver WithAddress(Hash256? address1) => + new PreCachedReadOnlyTraversalResolver(fullTrieStore, address1, inner.GetStorageTrieNodeResolver(address1)); + } } public readonly struct NodeKey : IEquatable, IHash64bit diff --git a/src/Nethermind/Nethermind.Trie/Pruning/IScopedReadOnlyTraversalProvider.cs b/src/Nethermind/Nethermind.Trie/Pruning/IScopedReadOnlyTraversalProvider.cs new file mode 100644 index 000000000000..3edf9bce3bc4 --- /dev/null +++ b/src/Nethermind/Nethermind.Trie/Pruning/IScopedReadOnlyTraversalProvider.cs @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; + +namespace Nethermind.Trie.Pruning; + +/// +/// Capability for a full that can hand out an address-scoped resolver +/// suitable for shared cached-node read-only traversal. Implemented by stores that opt in to +/// the fast path through their +/// wrappers. +/// +public interface IScopedReadOnlyTraversalProvider +{ + ITrieNodeResolver? GetReadOnlyTraversalResolver(Hash256? address); +} diff --git a/src/Nethermind/Nethermind.Trie/Pruning/ITrieNodeResolverSource.cs b/src/Nethermind/Nethermind.Trie/Pruning/ITrieNodeResolverSource.cs new file mode 100644 index 000000000000..815cfb11231f --- /dev/null +++ b/src/Nethermind/Nethermind.Trie/Pruning/ITrieNodeResolverSource.cs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Trie.Pruning; + +/// +/// Optional capability for trie resolvers that can provide a resolver for shared read-only traversal. +/// +public interface ITrieNodeResolverSource +{ + /// + /// Returns a resolver that may share immutable cached trie nodes for synchronous read-only traversal, + /// or when the wrapped resolver cannot preserve that behavior. + /// + ITrieNodeResolver? GetReadOnlyTraversalResolver(); +} diff --git a/src/Nethermind/Nethermind.Trie/Pruning/OverlayTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/OverlayTrieStore.cs index f1d7a437c470..c8968cc91d76 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/OverlayTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/OverlayTrieStore.cs @@ -13,32 +13,55 @@ namespace Nethermind.Trie.Pruning; /// On reset the base db provider is expected to clear any diff which causes this overlay trie store to no longer /// see overlaid keys. /// -public class OverlayTrieStore(IKeyValueStoreWithBatching keyValueStore, IReadOnlyTrieStore baseStore) : ITrieStore +public class OverlayTrieStore(IKeyValueStoreWithBatching keyValueStore, IReadOnlyTrieStore baseStore) + : ITrieStore, IScopedReadOnlyTraversalProvider { private readonly INodeStorage _nodeStorage = new NodeStorage(keyValueStore); + private readonly IReadOnlyTrieStore _baseStore = baseStore; - public void Dispose() => baseStore.Dispose(); + public void Dispose() => _baseStore.Dispose(); public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 hash) => // We always return Unknown even if baseStore return unknown, like archive node. - baseStore.FindCachedOrUnknown(address, in path, hash); + _baseStore.FindCachedOrUnknown(address, in path, hash); public byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => TryLoadRlp(address, in path, hash, flags) ?? throw new MissingTrieNodeException("Missing RLP node", address, path, hash); - public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => _nodeStorage.Get(address, in path, hash, flags) ?? baseStore.TryLoadRlp(address, in path, hash, flags); + public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => _nodeStorage.Get(address, in path, hash, flags) ?? _baseStore.TryLoadRlp(address, in path, hash, flags); - public bool HasRoot(Hash256 stateRoot) => _nodeStorage.Get(null, TreePath.Empty, stateRoot) is not null || baseStore.HasRoot(stateRoot); + public bool HasRoot(Hash256 stateRoot) => _nodeStorage.Get(null, TreePath.Empty, stateRoot) is not null || _baseStore.HasRoot(stateRoot); - public IDisposable BeginScope(BlockHeader? baseBlock) => baseStore.BeginScope(baseBlock); + public IDisposable BeginScope(BlockHeader? baseBlock) => _baseStore.BeginScope(baseBlock); public IScopedTrieStore GetTrieStore(Hash256? address) => new ScopedTrieStore(this, address); - public INodeStorage.KeyScheme Scheme => baseStore.Scheme; + public INodeStorage.KeyScheme Scheme => _baseStore.Scheme; public IBlockCommitter BeginBlockCommit(long blockNumber) => NullCommitter.Instance; // Write directly to _nodeStorage, which goes to db provider. public ICommitter BeginCommit(Hash256? address, TrieNode? root, WriteFlags writeFlags) => new RawScopedTrieStore.Committer(_nodeStorage, address, writeFlags); + + public ITrieNodeResolver? GetReadOnlyTraversalResolver(Hash256? address) => + new SharedOverlayTraversalResolver( + this, + address, + _baseStore.GetTrieStore(address).AsReadOnlyTraversal()); + + private sealed class SharedOverlayTraversalResolver( + OverlayTrieStore fullTrieStore, + Hash256? address, + ITrieNodeResolver baseReadResolver) : ReadOnlyTraversalResolverBase(fullTrieStore, address) + { + public override TrieNode FindCachedOrUnknown(in TreePath path, Hash256 hash) => + baseReadResolver.FindCachedOrUnknown(path, hash); + + protected override ITrieNodeResolver WithAddress(Hash256? address1) => + new SharedOverlayTraversalResolver( + fullTrieStore, + address1, + fullTrieStore._baseStore.GetTrieStore(address1).AsReadOnlyTraversal()); + } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTraversalResolverBase.cs b/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTraversalResolverBase.cs new file mode 100644 index 000000000000..f5061989d448 --- /dev/null +++ b/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTraversalResolverBase.cs @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Crypto; + +namespace Nethermind.Trie.Pruning; + +/// +/// Common skeleton for shared read-only traversal resolvers. Forwards LoadRlp/TryLoadRlp/Scheme +/// to the wrapping full store (so layered behavior like pre-block caching or witness capture is +/// preserved on the read path) and asks the derived class for the cached-node lookup and the +/// per-address rebuild. +/// +public abstract class ReadOnlyTraversalResolverBase(IScopableTrieStore fullTrieStore, Hash256? address) : ITrieNodeResolver +{ + protected Hash256? Address => address; + + public abstract TrieNode FindCachedOrUnknown(in TreePath path, Hash256 hash); + + public byte[]? LoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => + fullTrieStore.LoadRlp(address, path, hash, flags); + + public byte[]? TryLoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => + fullTrieStore.TryLoadRlp(address, path, hash, flags); + + public INodeStorage.KeyScheme Scheme => fullTrieStore.Scheme; + + public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address1) => + address1 == address ? this : WithAddress(address1); + + protected abstract ITrieNodeResolver WithAddress(Hash256? address1); +} diff --git a/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs index 99bea61a0b6a..ae29a8576461 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs @@ -10,7 +10,7 @@ namespace Nethermind.Trie.Pruning /// /// Safe to be reused for the same wrapped store. /// - public class ReadOnlyTrieStore(TrieStore trieStore) : IReadOnlyTrieStore + public class ReadOnlyTrieStore(TrieStore trieStore) : IReadOnlyTrieStore, IScopedReadOnlyTraversalProvider { private readonly TrieStore _trieStore = trieStore ?? throw new ArgumentNullException(nameof(trieStore)); public INodeStorage.KeyScheme Scheme => _trieStore.Scheme; @@ -29,7 +29,7 @@ public byte[] LoadRlp(Hash256? address, in TreePath treePath, Hash256 hash, Read public IDisposable BeginScope(BlockHeader? baseBlock) => new Reactive.AnonymousDisposable(() => { }); // Noop - public IScopedTrieStore GetTrieStore(Hash256? address) => new ScopedReadOnlyTrieStore(this, address); + public IScopedTrieStore GetTrieStore(Hash256? address) => new ScopedTrieStore(this, address); public bool HasRoot(Hash256 stateRoot) => _trieStore.HasRoot(stateRoot); @@ -37,22 +37,17 @@ public byte[] LoadRlp(Hash256? address, in TreePath treePath, Hash256 hash, Read public void Dispose() { } - private class ScopedReadOnlyTrieStore(ReadOnlyTrieStore fullTrieStore, Hash256? address) : IScopedTrieStore - { - public TrieNode FindCachedOrUnknown(in TreePath path, Hash256 hash) => - fullTrieStore.FindCachedOrUnknown(address, path, hash); - - public byte[]? LoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => - fullTrieStore.LoadRlp(address, path, hash, flags); - - public byte[]? TryLoadRlp(in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => fullTrieStore.TryLoadRlp(address, path, hash, flags); + public ITrieNodeResolver? GetReadOnlyTraversalResolver(Hash256? address) => + new SharedReadOnlyTraversalResolver(this, address); - public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address1) => - address1 == address ? this : new ScopedReadOnlyTrieStore(fullTrieStore, address1); - - public INodeStorage.KeyScheme Scheme => fullTrieStore.Scheme; + private sealed class SharedReadOnlyTraversalResolver(ReadOnlyTrieStore fullTrieStore, Hash256? address) + : ReadOnlyTraversalResolverBase(fullTrieStore, address) + { + public override TrieNode FindCachedOrUnknown(in TreePath path, Hash256 hash) => + fullTrieStore._trieStore.FindCachedOrUnknownShared(Address, path, hash); - public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => NullCommitter.Instance; + protected override ITrieNodeResolver WithAddress(Hash256? address1) => + new SharedReadOnlyTraversalResolver(fullTrieStore, address1); } } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/ScopedTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/ScopedTrieStore.cs index 249014b87d28..1af55ee104ee 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/ScopedTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/ScopedTrieStore.cs @@ -6,7 +6,8 @@ namespace Nethermind.Trie.Pruning; -public sealed class ScopedTrieStore(IScopableTrieStore fullTrieStore, Hash256? address) : IScopedTrieStore +public sealed class ScopedTrieStore(IScopableTrieStore fullTrieStore, Hash256? address) + : IScopedTrieStore, ITrieNodeResolverSource { public TrieNode FindCachedOrUnknown(in TreePath path, Hash256 hash) => fullTrieStore.FindCachedOrUnknown(address, path, hash); @@ -24,4 +25,9 @@ public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address1) => public ICommitter BeginCommit(TrieNode? root, WriteFlags writeFlags = WriteFlags.None) => fullTrieStore.BeginCommit(address, root, writeFlags); + + public ITrieNodeResolver? GetReadOnlyTraversalResolver() => + fullTrieStore is IScopedReadOnlyTraversalProvider provider + ? provider.GetReadOnlyTraversalResolver(address) + : null; } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieNodeResolverExtensions.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieNodeResolverExtensions.cs new file mode 100644 index 000000000000..a2f2aa995645 --- /dev/null +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieNodeResolverExtensions.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Trie.Pruning; + +internal static class TrieNodeResolverExtensions +{ + public static ITrieNodeResolver AsReadOnlyTraversal(this ITrieNodeResolver self) => + self is ITrieNodeResolverSource source && source.GetReadOnlyTraversalResolver() is { } shared + ? shared + : self; +} diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs index 29ae6c428a35..b094b1bb00c3 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs @@ -318,6 +318,11 @@ long blockNumber shard.IncrementMemory(node); } + if (cachedNodeCopy.IsSealed && cachedNodeCopy.HasRlp) + { + cachedNodeCopy.PreDecodeChildrenIfBranch(ref path); + } + return cachedNodeCopy; } @@ -531,6 +536,40 @@ internal TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash25 private TrieNode FindCachedOrUnknown(TrieStoreDirtyNodesCache.Key key, bool isReadOnly) => isReadOnly ? DirtyNodesFromCachedRlpOrUnknown(key) : DirtyNodesFindCachedOrUnknown(key); + internal TrieNode FindCachedOrUnknownShared(Hash256? address, in TreePath path, Hash256 hash) + { + TrieStoreDirtyNodesCache.Key key = new(address, path, hash); + return Volatile.Read(ref _commitBuffer) is { } commitBuffer + ? commitBuffer.FindCachedOrUnknownShared(key) + : FindCachedOrUnknownShared(key); + } + + private TrieNode FindCachedOrUnknownShared(in TrieStoreDirtyNodesCache.Key key) => + DirtyNodesTryGetValue(key, out TrieNode? cached) + ? ShareOrCloneForReadOnly(key, cached) + : new TrieNode(NodeType.Unknown, key.Keccak); + + private TrieNode ShareOrCloneForReadOnly(in TrieStoreDirtyNodesCache.Key key, TrieNode cached) + { + Metrics.LoadedFromCacheNodesCount++; + + Debug.Assert(cached.Keccak == key.Keccak, "Cache key/Keccak mismatch."); + if (!cached.HasRlp) + { + IncrementFallbackNotShareableCount(); + return new TrieNode(NodeType.Unknown, key.Keccak); + } + + if (cached.IsSealed) + { + IncrementSharedNodeHitCount(); + return cached; + } + + IncrementFallbackNotShareableCount(); + return CloneForReadOnly(key, cached); + } + // Used only in tests public void Dump() { @@ -1037,6 +1076,41 @@ private void FlushNonBlockingBuffer() private long _totalCachedNodesCount; private long _dirtyNodesCount; + // Test-only read-only traversal counters. They are disabled until tests read one of the + // properties below, keeping the production read-only traversal path free of Interlocked counting. + private bool _collectReadOnlyTraversalStats; + private long _cloneForReadOnlyCount; + private long _fallbackNotShareableCount; + private long _sharedNodeHitCount; + + internal long CloneForReadOnlyCount => EnableReadOnlyTraversalStatsAndRead(ref _cloneForReadOnlyCount); + internal long FallbackNotShareableCount => EnableReadOnlyTraversalStatsAndRead(ref _fallbackNotShareableCount); + internal long SharedNodeHitCount => EnableReadOnlyTraversalStatsAndRead(ref _sharedNodeHitCount); + + private long EnableReadOnlyTraversalStatsAndRead(ref long counter) + { + _collectReadOnlyTraversalStats = true; + return Interlocked.Read(ref counter); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void IncrementCloneForReadOnlyCount() + { + if (_collectReadOnlyTraversalStats) Interlocked.Increment(ref _cloneForReadOnlyCount); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void IncrementFallbackNotShareableCount() + { + if (_collectReadOnlyTraversalStats) Interlocked.Increment(ref _fallbackNotShareableCount); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void IncrementSharedNodeHitCount() + { + if (_collectReadOnlyTraversalStats) Interlocked.Increment(ref _sharedNodeHitCount); + } + private int _committedNodesCount; private int _persistedNodesCount; @@ -1405,13 +1479,8 @@ private class TrieKeyValueStore(TrieStore trieStore) : IReadOnlyKeyValueStore public bool HasRoot(Hash256 stateRoot) { if (stateRoot == Keccak.EmptyTreeHash) return true; - TrieNode node = FindCachedOrUnknown(null, TreePath.Empty, stateRoot, true); - if (node.NodeType == NodeType.Unknown) - { - return TryLoadRlp(null, TreePath.Empty, node.Keccak!) is not null; - } - - return true; + TrieStoreDirtyNodesCache.Key key = new(null, TreePath.Empty, stateRoot); + return HasCachedRlp(key) || TryLoadRlp(null, TreePath.Empty, stateRoot) is not null; } public bool HasRoot(Hash256 stateRoot, long blockNumber) @@ -1712,10 +1781,62 @@ public TrieNode FindCachedOrUnknown(TrieStoreDirtyNodesCache.Key key, bool isRea return hasInBuffer ? bufferNode : bufferShard.FindCachedOrUnknown(key); } + + public TrieNode FindCachedOrUnknownShared(TrieStoreDirtyNodesCache.Key key) + { + int shardIdx = _trieStore.GetNodeShardIdx(key.Path, key.Keccak); + TrieStoreDirtyNodesCache bufferShard = _dirtyNodesBuffer[shardIdx]; + TrieStoreDirtyNodesCache mainShard = _trieStore._dirtyNodes[shardIdx]; + + if (bufferShard.TryGetValue(key, out TrieNode bufferNode)) + { + return _trieStore.ShareOrCloneForReadOnly(key, bufferNode); + } + + if (mainShard.TryGetRecord(key, out TrieStoreDirtyNodesCache.NodeRecord nodeRecord)) + { + if (CanImportFromMainCache(nodeRecord)) + { + TrieStoreDirtyNodesCache.NodeRecord imported = bufferShard.GetOrAdd(key, new TrieStoreDirtyNodesCache.NodeRecord(nodeRecord.Node, -1)); + return _trieStore.ShareOrCloneForReadOnly(key, imported.Node); + } + + return _trieStore.ShareOrCloneForReadOnly(key, nodeRecord.Node); + } + + return new TrieNode(NodeType.Unknown, key.Keccak); + } + + private bool CanImportFromMainCache(TrieStoreDirtyNodesCache.NodeRecord nodeRecord) => + !nodeRecord.Node.IsPersisted || nodeRecord.LastCommit >= _minCommitBlockNumber; + + public bool HasCachedRlp(TrieStoreDirtyNodesCache.Key key) + { + int shardIdx = _trieStore.GetNodeShardIdx(key.Path, key.Keccak); + return TrieStore.HasCachedRlp(_dirtyNodesBuffer[shardIdx], key) || TrieStore.HasCachedRlp(_trieStore._dirtyNodes[shardIdx], key); + } + } + + private bool HasCachedRlp(in TrieStoreDirtyNodesCache.Key key) => + Volatile.Read(ref _commitBuffer) is { } commitBuffer + ? commitBuffer.HasCachedRlp(key) + : HasCachedRlp(GetDirtyNodeShard(key), key); + + private static bool HasCachedRlp(TrieStoreDirtyNodesCache shard, TrieStoreDirtyNodesCache.Key key) + { + if (!shard.TryGetRecord(key, out TrieStoreDirtyNodesCache.NodeRecord nodeRecord) || nodeRecord.Node.FullRlp.IsNull) + { + return false; + } + + Metrics.LoadedFromCacheNodesCount++; + return true; } internal TrieNode CloneForReadOnly(in TrieStoreDirtyNodesCache.Key key, TrieNode node) { + IncrementCloneForReadOnlyCount(); + CappedArray fullRlp = node!.FullRlp; if (fullRlp.IsNull) { diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.cs b/src/Nethermind/Nethermind.Trie/TrieNode.cs index b5c45ecf5bd9..31e6c531a8dd 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Runtime.Intrinsics.X86; using System.Threading; using System.Threading.Tasks; @@ -23,6 +24,7 @@ namespace Nethermind.Trie { + [StructLayout(LayoutKind.Sequential)] public sealed partial class TrieNode { internal const int BranchesCount = 16; @@ -44,9 +46,20 @@ public sealed partial class TrieNode // not atomically readable on x64. Split into two 8-byte fields that are // individually atomic, with a sequence counter to detect concurrent writes. private byte[]? _rlpArray; - private ulong _rlpSeqAndLength; // bits 0-31: length, bits 32-63: sequence (even = stable, odd = writing) + private ulong _rlpSeqAndLength; // normal: bits 0-31 length, 32-63 sequence. slice: bit 63, bits 0-31 length, bits 32-62 offset. private INodeData? _nodeData; + // In normal mode, the sequence counter shares bit 63 with the slice flag. After about + // 2^30 completed writes to one node, doneSeq reaches 0x80000000 and IsRlpSlice returns + // true for a normal value. Readers still get offset 0 and the real length, and WriteRlp + // resets the sequence on the next write so it cannot advance to a wrong non-zero offset. + private const ulong RlpSliceFlag = 1UL << 63; + private const ulong RlpSliceLengthMask = 0xFFFFFFFFUL; + private const int RlpSliceOffsetShift = 32; + private const ulong RlpSliceOffsetMask = 0x7FFFFFFFUL; + + private static bool IsRlpSlice(ulong value) => (value & RlpSliceFlag) != 0; + /// /// Atomically read _rlp using seqlock: retry if a concurrent write is detected. /// Memory barriers ensure ARM64 correctness (matching SeqlockCache/KeccakCache patterns). @@ -59,6 +72,24 @@ private CappedArray ReadRlp() while (true) { seqBefore = Volatile.Read(ref _rlpSeqAndLength); + if (IsRlpSlice(seqBefore)) + { + array = Volatile.Read(ref _rlpArray); + seqAfter = Volatile.Read(ref _rlpSeqAndLength); + if (seqBefore == seqAfter) + { + return array is null + ? default + : new CappedArray( + array, + (int)((seqBefore >> RlpSliceOffsetShift) & RlpSliceOffsetMask), + (int)(seqBefore & RlpSliceLengthMask)); + } + + spin.SpinOnce(); + continue; + } + if ((seqBefore >> 32 & 1) != 0) { spin.SpinOnce(); continue; } if (!Sse.IsSupported) Interlocked.MemoryBarrier(); array = _rlpArray; @@ -75,7 +106,7 @@ private CappedArray ReadRlp() /// Atomically write _rlp using seqlock: odd sequence signals write-in-progress. /// CAS on even sequences only — if another writer is active (odd), spin until it completes. /// Last writer wins: all writers write the same resolved data for a given node. - /// Sequence uses bits 1-31 (31 bits, ~2 billion writes before wrap); bit 0 is the lock flag. + /// Sequence uses bits 1-31; bit 0 is the lock flag and bit 31 overlaps the slice flag. /// [MethodImpl(MethodImplOptions.NoInlining)] // CAS dominates latency; avoid code bloat at 5+ call sites internal void WriteRlp(CappedArray value) @@ -84,7 +115,9 @@ internal void WriteRlp(CappedArray value) while (true) { ulong current = Volatile.Read(ref _rlpSeqAndLength); - uint seq = (uint)(current >> 32); + // If a normal-mode sequence reached the slice flag bit, reset before the next + // completed write can publish a non-zero slice offset. + uint seq = IsRlpSlice(current) ? 0 : (uint)(current >> 32); if ((seq & 1) != 0) { // Another writer is active — spin until it completes @@ -98,7 +131,7 @@ internal void WriteRlp(CappedArray value) Volatile.Write(ref _rlpArray, value.UnderlyingArray); // Advance sequence by 2 and clear lock bit (even), store final length uint doneSeq = (seq + 2) & 0xFFFFFFFE; - Volatile.Write(ref _rlpSeqAndLength, (ulong)doneSeq << 32 | (uint)value.Length); + Volatile.Write(ref _rlpSeqAndLength, CreateRlpMetadata(value, doneSeq)); return; } spin.SpinOnce(); // CAS failed — another writer raced; back off before retry @@ -112,9 +145,26 @@ internal void WriteRlp(CappedArray value) private void InitRlp(CappedArray value) { _rlpArray = value.UnderlyingArray; - _rlpSeqAndLength = (uint)value.Length; + _rlpSeqAndLength = CreateRlpMetadata(value); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void InitRlpSlice(byte[] parentRlp, int offset, int length) + { + _rlpArray = parentRlp; + _rlpSeqAndLength = CreateRlpSliceMetadata(offset, length); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong CreateRlpMetadata(CappedArray value, uint normalSeq = 0) => + value.Offset == 0 + ? ((ulong)normalSeq << 32) | (uint)value.Length + : CreateRlpSliceMetadata(value.Offset, value.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong CreateRlpSliceMetadata(int offset, int length) => + RlpSliceFlag | ((ulong)(uint)offset << RlpSliceOffsetShift) | (uint)length; + /// /// Sealed node is the one that is already immutable except for reference counting and resolving existing data /// @@ -359,6 +409,16 @@ public TrieNode(NodeType nodeType, Hash256 keccak, CappedArray rlp) } } + private TrieNode(CappedArray parentRlp, int offset, int length) + { + _nodeData = null; + // offset is relative to parentRlp's logical view; InitRlpSlice needs a backing-array index. + InitRlpSlice(parentRlp.UnderlyingArray!, parentRlp.Offset + offset, length); + } + + private static TrieNode CreateInlineChild(CappedArray parentRlp, int offset, int length) => + new(parentRlp, offset, length); + private INodeData CreateNodeData(NodeType nodeType) => nodeType switch { NodeType.Branch => new BranchData(), @@ -518,6 +578,46 @@ private bool DecodeRlp(ValueRlpStream rlpStream, ICappedArrayPool bufferPool, ou return true; } + internal void PreDecodeChildrenIfBranch(ref TreePath path) + { + if (_nodeData is not BranchData || !IsSealed) + { + return; + } + + CappedArray rlp = ReadRlp(); + if (rlp.IsNull) + { + return; + } + + int originalPathLength = path.Length; + try + { + ValueRlpStream rlpStream = new(rlp); + rlpStream.Reset(); + rlpStream.SkipLength(); + path.AppendMut(0); + + for (int i = 0; i < BranchesCount; i++) + { + path.SetLast(i); + ref object data = ref _nodeData[i]; + if (Volatile.Read(ref data) is not null) + { + rlpStream.SkipItem(); + continue; + } + + PublishChild(ref data, DecodeChildReference(rlp, ref rlpStream)); + } + } + finally + { + path.TruncateMut(originalPathLength); + } + } + public void ResolveKey(ITrieNodeResolver tree, ref TreePath path, ICappedArrayPool? bufferPool = null, bool canBeParallel = true) @@ -851,7 +951,8 @@ public long GetMemorySize(bool recursive) { int keccakSize = Keccak is null ? MemorySizes.RefSize : MemorySizes.RefSize + Hash256.MemorySize; CappedArray rlp = ReadRlp(); - long rlpSize = MemorySizes.RefSize + (rlp.IsNotNull ? MemorySizes.ArrayOverhead + rlp.UnderlyingLength : 0); + bool isRlpSlice = IsRlpSlice(Volatile.Read(ref _rlpSeqAndLength)); + long rlpSize = MemorySizes.RefSize + (rlp.IsNotNull && !isRlpSlice ? MemorySizes.ArrayOverhead + rlp.UnderlyingLength : 0); long dataSize = MemorySizes.RefSize + (_nodeData?.MemorySize ?? 0); int objectOverhead = MemorySizes.ObjectHeaderMethodTable; int blockAndFlagsSize = sizeof(long); @@ -1273,6 +1374,36 @@ private void SeekChildNotNull(ref ValueRlpStream rlpStream, int index) } } + // Slot publication primitive paired with Volatile.Read on _nodeData[i] readers. + // On x64 the read is a plain mov and the CAS is a locked op; on ARM64 the read is ldar + // (load-acquire) and the CAS provides store-release - readers see fully-constructed children. + private static object PublishChild(ref object slot, object decoded) + { + object winner = Interlocked.CompareExchange(ref slot, decoded, null); + return winner ?? decoded; + } + + private static object DecodeChildReference(CappedArray rlp, ref ValueRlpStream rlpStream) + { + int prefix = rlpStream.ReadByte(); + switch (prefix) + { + case 0: + case 128: + return _nullNode; + case 160: + rlpStream.Position--; + return rlpStream.DecodeKeccak()!; + default: + rlpStream.Position--; + int offset = rlpStream.Position; + int length = rlpStream.PeekNextRlpLength(); + TrieNode child = CreateInlineChild(rlp, offset, length); + rlpStream.SkipBytes(length); + return child; + } + } + private object? ResolveChildWithChildPath(ITrieNodeResolver tree, ref TreePath childPath, int i) { object? childOrRef; @@ -1280,45 +1411,22 @@ private void SeekChildNotNull(ref ValueRlpStream rlpStream, int index) ref object data = ref _nodeData[i]; if (rlp.IsNull) { - childOrRef = data; + childOrRef = Volatile.Read(ref data); } else { - childOrRef = data; + childOrRef = Volatile.Read(ref data); if (childOrRef is null) { // Allows to load children in parallel ValueRlpStream rlpStream = new(rlp); SeekChild(ref rlpStream, i); - int prefix = rlpStream.ReadByte(); - - switch (prefix) + childOrRef = DecodeChildReference(rlp, ref rlpStream); + if (childOrRef is Hash256 keccak) { - case 0: - case 128: - { - data = childOrRef = _nullNode; - break; - } - case 160: - { - rlpStream.Position--; - Hash256 keccak = rlpStream.DecodeKeccak(); - - TrieNode child = tree.FindCachedOrUnknown(childPath, keccak); - data = childOrRef = child; - - break; - } - default: - { - rlpStream.Position--; - ReadOnlySpan fullRlp = rlpStream.PeekNextItem(); - TrieNode child = new(NodeType.Unknown, fullRlp.ToArray()); - data = childOrRef = child; - break; - } + childOrRef = tree.FindCachedOrUnknown(childPath, keccak); } + childOrRef = PublishChild(ref data, childOrRef); } } @@ -1381,9 +1489,10 @@ internal int ResolveAllChildBranch(ITrieNodeResolver tree, ref TreePath path, Sp } default: { - ReadOnlySpan fullRlp = rlpStream.PeekNextItem(); - TrieNode child = new(NodeType.Unknown, fullRlp.ToArray()); - rlpStream.SkipItem(); + int offset = rlpStream.Position; + int length = rlpStream.PeekNextRlpLength(); + TrieNode child = CreateInlineChild(rlp, offset, length); + rlpStream.SkipBytes(length); chCount++; output[i] = child; break; @@ -1401,11 +1510,16 @@ internal void UnresolveChild(int i) ref object data = ref _nodeData[i]; if (IsPersisted) { - data = null; + object observed = Volatile.Read(ref data); + if (observed is not null) + { + Interlocked.CompareExchange(ref data, null, observed); + } } else { - if (data is TrieNode childNode) + object observed = Volatile.Read(ref data); + if (observed is TrieNode childNode) { if (!childNode.IsPersisted) { @@ -1413,7 +1527,7 @@ internal void UnresolveChild(int i) } else if (childNode.Keccak is not null) // if not by value node { - data = childNode.Keccak; + Interlocked.CompareExchange(ref data, childNode.Keccak, childNode); } } } @@ -1437,11 +1551,11 @@ public ref struct ChildIterator(TrieNode node) ref object data = ref node._nodeData[i]; if (rlp.IsNull) { - childOrRef = data; + childOrRef = Volatile.Read(ref data); } else { - childOrRef = data; + childOrRef = Volatile.Read(ref data); if (childOrRef is null) { if (_currentStreamIndex.HasValue && _currentStreamIndex <= i) @@ -1468,37 +1582,13 @@ public ref struct ChildIterator(TrieNode node) _currentStreamIndex = i; } - int prefix = _rlpStream.ReadByte(); - - switch (prefix) + childOrRef = DecodeChildReference(rlp, ref _rlpStream); + if (childOrRef is Hash256 keccak) { - case 0: - case 128: - { - data = childOrRef = _nullNode; - _currentStreamIndex++; - break; - } - case 160: - { - _rlpStream.Position--; - Hash256 keccak = _rlpStream.DecodeKeccak(); - _currentStreamIndex++; - - TrieNode child = tree.FindCachedOrUnknown(childPath, keccak); - data = childOrRef = child; - - break; - } - default: - { - _rlpStream.Position--; - ReadOnlySpan fullRlp = _rlpStream.PeekNextItem(); - TrieNode child = new(NodeType.Unknown, fullRlp.ToArray()); - data = childOrRef = child; - break; - } + childOrRef = tree.FindCachedOrUnknown(childPath, keccak); } + childOrRef = PublishChild(ref data, childOrRef); + _currentStreamIndex++; } } diff --git a/src/Nethermind/Nethermind.Trie/TrieNodeResolverWithReadFlags.cs b/src/Nethermind/Nethermind.Trie/TrieNodeResolverWithReadFlags.cs index fa9fd6efe732..1cc335f2e2ed 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNodeResolverWithReadFlags.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNodeResolverWithReadFlags.cs @@ -7,7 +7,7 @@ namespace Nethermind.Trie; -public class TrieNodeResolverWithReadFlags(ITrieNodeResolver baseResolver, ReadFlags defaultFlags) : ITrieNodeResolver +public class TrieNodeResolverWithReadFlags(ITrieNodeResolver baseResolver, ReadFlags defaultFlags) : ITrieNodeResolver, ITrieNodeResolverSource { private readonly ITrieNodeResolver _baseResolver = baseResolver; private readonly ReadFlags _defaultFlags = defaultFlags; @@ -34,7 +34,13 @@ public class TrieNodeResolverWithReadFlags(ITrieNodeResolver baseResolver, ReadF return _baseResolver.LoadRlp(treePath, hash, _defaultFlags); } - public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256 address) => new TrieNodeResolverWithReadFlags(_baseResolver.GetStorageTrieNodeResolver(address), _defaultFlags); + public ITrieNodeResolver GetStorageTrieNodeResolver(Hash256? address) => new TrieNodeResolverWithReadFlags(_baseResolver.GetStorageTrieNodeResolver(address), _defaultFlags); public INodeStorage.KeyScheme Scheme => _baseResolver.Scheme; + + public ITrieNodeResolver? GetReadOnlyTraversalResolver() => + _baseResolver is ITrieNodeResolverSource source + && source.GetReadOnlyTraversalResolver() is ITrieNodeResolver readOnlyResolver + ? new TrieNodeResolverWithReadFlags(readOnlyResolver, _defaultFlags) + : null; }